diff --git a/.gitattributes b/.gitattributes
index aaa7367083c98796ddaec1a9a2cf1fc5e95cc237..da8664dbb511498bb996ab5288863b767a15c295 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -53,3 +53,5 @@ MOSS_Speaker-RoPE/tests/speaker_embed_postnorm.png filter=lfs diff=lfs merge=lfs
 MOSS_Speaker-RoPE/tests/speaker_spkw5_heatmap.png filter=lfs diff=lfs merge=lfs -text
 MOSS_Speaker-RoPE/tests/speaker_embed_heatmap_norm.png filter=lfs diff=lfs merge=lfs -text
 MOSS_Speaker-RoPE/data/train_40min_new.jsonl filter=lfs diff=lfs merge=lfs -text
+MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..90da567b458e2df298e5ce54139e0704e4b1aa24
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/config.json
@@ -0,0 +1,250 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_rotary_dim": 64,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 1.0,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "partial_rotary_factor": 0.5,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/generation_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/processor_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/trainer_state.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..9602b7edb8a44c3d91563452faeca48e6aca2081
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1200/trainer_state.json
@@ -0,0 +1,1714 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9942004971002486,
+  "eval_steps": 500,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 1147.55224609375,
+      "learning_rate": 6.611570247933885e-06,
+      "loss": 11.37186279296875,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 246.35504150390625,
+      "learning_rate": 1.487603305785124e-05,
+      "loss": 9.2022216796875,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 79.90882110595703,
+      "learning_rate": 2.3140495867768598e-05,
+      "loss": 7.6917167663574215,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 158.79872131347656,
+      "learning_rate": 3.1404958677685955e-05,
+      "loss": 6.138848114013672,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 48.520408630371094,
+      "learning_rate": 3.9669421487603306e-05,
+      "loss": 5.638393402099609,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 14.776691436767578,
+      "learning_rate": 4.793388429752066e-05,
+      "loss": 5.221096801757812,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 12.125936508178711,
+      "learning_rate": 5.619834710743802e-05,
+      "loss": 4.968892288208008,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 14.540782928466797,
+      "learning_rate": 6.446280991735537e-05,
+      "loss": 5.3284343719482425,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 13.903443336486816,
+      "learning_rate": 7.272727272727273e-05,
+      "loss": 5.018967819213867,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 12.485719680786133,
+      "learning_rate": 8.099173553719009e-05,
+      "loss": 4.600848770141601,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 44.14723205566406,
+      "learning_rate": 8.925619834710744e-05,
+      "loss": 4.335272979736328,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 9.01069164276123,
+      "learning_rate": 9.75206611570248e-05,
+      "loss": 4.316203308105469,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 16.169389724731445,
+      "learning_rate": 0.00010578512396694216,
+      "loss": 4.197259140014649,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 6.058357238769531,
+      "learning_rate": 0.0001140495867768595,
+      "loss": 4.562212371826172,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 5.126786231994629,
+      "learning_rate": 0.00012231404958677685,
+      "loss": 4.340535354614258,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 12.424266815185547,
+      "learning_rate": 0.00013057851239669423,
+      "loss": 3.9918304443359376,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 8.042577743530273,
+      "learning_rate": 0.0001388429752066116,
+      "loss": 4.087551879882812,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 8.642739295959473,
+      "learning_rate": 0.00014710743801652894,
+      "loss": 4.22486686706543,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 8.129426002502441,
+      "learning_rate": 0.00015537190082644627,
+      "loss": 3.8094669342041017,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 10.393760681152344,
+      "learning_rate": 0.00016363636363636366,
+      "loss": 3.687621307373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 23.797346115112305,
+      "learning_rate": 0.00017190082644628102,
+      "loss": 4.8999065399169925,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 7.251102924346924,
+      "learning_rate": 0.00018016528925619835,
+      "loss": 4.111215972900391,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 7.632305145263672,
+      "learning_rate": 0.0001884297520661157,
+      "loss": 3.88475341796875,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 2.708984851837158,
+      "learning_rate": 0.0001966942148760331,
+      "loss": 3.9735763549804686,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 3.5484111309051514,
+      "learning_rate": 0.00019999623426388962,
+      "loss": 3.911006546020508,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 3.580739974975586,
+      "learning_rate": 0.0001999732224591804,
+      "loss": 3.6748367309570313,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 3.1602091789245605,
+      "learning_rate": 0.0001999292957337329,
+      "loss": 4.212306213378906,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 2.242072582244873,
+      "learning_rate": 0.0001998644632772477,
+      "loss": 3.8557418823242187,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 3.2909867763519287,
+      "learning_rate": 0.00019977873865301363,
+      "loss": 3.953845977783203,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 2.695601224899292,
+      "learning_rate": 0.00019967213979507018,
+      "loss": 3.7311931610107423,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 3.831533670425415,
+      "learning_rate": 0.00019954468900445566,
+      "loss": 3.88598518371582,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 4.886938571929932,
+      "learning_rate": 0.00019939641294454172,
+      "loss": 3.8442028045654295,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 3.451791524887085,
+      "learning_rate": 0.00019922734263545517,
+      "loss": 3.9103469848632812,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.0034420490264893,
+      "learning_rate": 0.00019903751344758848,
+      "loss": 4.032486343383789,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 2.849339723587036,
+      "learning_rate": 0.00019882696509420007,
+      "loss": 3.407342529296875,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 2.095475435256958,
+      "learning_rate": 0.00019859574162310608,
+      "loss": 3.7116188049316405,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 3.148158073425293,
+      "learning_rate": 0.0001983438914074654,
+      "loss": 3.7624229431152343,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 2.4163601398468018,
+      "learning_rate": 0.00019807146713565955,
+      "loss": 3.439311218261719,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 2.7623701095581055,
+      "learning_rate": 0.00019777852580027042,
+      "loss": 3.8211971282958985,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 3.145005226135254,
+      "learning_rate": 0.00019746512868615656,
+      "loss": 3.5431480407714844,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 2.633315324783325,
+      "learning_rate": 0.00019713134135763256,
+      "loss": 3.784006881713867,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 2.0188381671905518,
+      "learning_rate": 0.00019677723364475237,
+      "loss": 3.7024520874023437,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 2.1343772411346436,
+      "learning_rate": 0.00019640287962870062,
+      "loss": 3.7561168670654297,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 1.9709585905075073,
+      "learning_rate": 0.0001960083576262943,
+      "loss": 3.6580154418945314,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 2.0804245471954346,
+      "learning_rate": 0.0001955937501735987,
+      "loss": 3.379087448120117,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 1.890079379081726,
+      "learning_rate": 0.0001951591440086602,
+      "loss": 3.5966243743896484,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 2.0973691940307617,
+      "learning_rate": 0.0001947046300533604,
+      "loss": 3.4583953857421874,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 2.4010658264160156,
+      "learning_rate": 0.00019423030339439464,
+      "loss": 3.3876201629638674,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 2.172189235687256,
+      "learning_rate": 0.00019373626326337946,
+      "loss": 3.7462398529052736,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 1.9334107637405396,
+      "learning_rate": 0.00019322261301609286,
+      "loss": 3.841419219970703,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 2.0680906772613525,
+      "learning_rate": 0.00019268946011085158,
+      "loss": 3.4502696990966797,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 2.5843758583068848,
+      "learning_rate": 0.00019213691608603047,
+      "loss": 3.404983139038086,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 1.888687252998352,
+      "learning_rate": 0.00019156509653672782,
+      "loss": 3.549253463745117,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 1.8593695163726807,
+      "learning_rate": 0.00019097412109058247,
+      "loss": 3.422663116455078,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 2.036330461502075,
+      "learning_rate": 0.00019036411338274703,
+      "loss": 3.7582332611083986,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 1.8483343124389648,
+      "learning_rate": 0.00018973520103002277,
+      "loss": 3.517898941040039,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 2.0232744216918945,
+      "learning_rate": 0.00018908751560416142,
+      "loss": 3.521917724609375,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 3.3193116188049316,
+      "learning_rate": 0.00018842119260433982,
+      "loss": 3.4880836486816404,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 2.5043585300445557,
+      "learning_rate": 0.0001877363714288126,
+      "loss": 3.446257400512695,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 1.9308786392211914,
+      "learning_rate": 0.00018703319534574956,
+      "loss": 3.3181568145751954,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 1.857397198677063,
+      "learning_rate": 0.00018631181146326305,
+      "loss": 3.219373321533203,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.9737612009048462,
+      "learning_rate": 0.00018557237069863222,
+      "loss": 3.5038528442382812,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 4.4995598793029785,
+      "learning_rate": 0.00018481502774673044,
+      "loss": 3.3514575958251953,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 2.948864221572876,
+      "learning_rate": 0.00018403994104766212,
+      "loss": 3.13072509765625,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 2.1394331455230713,
+      "learning_rate": 0.00018324727275361645,
+      "loss": 3.5682483673095704,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 3.813765287399292,
+      "learning_rate": 0.00018243718869494408,
+      "loss": 3.478915786743164,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.6351795196533203,
+      "learning_rate": 0.00018160985834546475,
+      "loss": 3.307242584228516,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 2.2016730308532715,
+      "learning_rate": 0.00018076545478701235,
+      "loss": 3.5482852935791014,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 2.1184046268463135,
+      "learning_rate": 0.00017990415467322534,
+      "loss": 3.51483039855957,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 1.704121470451355,
+      "learning_rate": 0.00017902613819258985,
+      "loss": 3.6264686584472656,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 1.926048994064331,
+      "learning_rate": 0.00017813158903074342,
+      "loss": 3.472669219970703,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 2.019219160079956,
+      "learning_rate": 0.00017722069433204687,
+      "loss": 3.2899856567382812,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 2.0369884967803955,
+      "learning_rate": 0.00017629364466043273,
+      "loss": 3.427472686767578,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 1.7189043760299683,
+      "learning_rate": 0.0001753506339595384,
+      "loss": 3.403704833984375,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 2.494591236114502,
+      "learning_rate": 0.00017439185951213202,
+      "loss": 3.5159603118896485,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 1.7678285837173462,
+      "learning_rate": 0.00017341752189883983,
+      "loss": 3.119024467468262,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 2.24629807472229,
+      "learning_rate": 0.0001724278249561838,
+      "loss": 3.525630569458008,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 1.8001821041107178,
+      "learning_rate": 0.0001714229757339379,
+      "loss": 3.541637420654297,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 2.8331096172332764,
+      "learning_rate": 0.0001704031844518121,
+      "loss": 3.3902435302734375,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 2.052628755569458,
+      "learning_rate": 0.00016936866445547353,
+      "loss": 3.218609619140625,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 2.2566065788269043,
+      "learning_rate": 0.00016831963217191334,
+      "loss": 3.5105533599853516,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.5952314138412476,
+      "learning_rate": 0.0001672563070641688,
+      "loss": 3.253592681884766,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 1.6297531127929688,
+      "learning_rate": 0.000166178911585411,
+      "loss": 3.176485061645508,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 1.6768420934677124,
+      "learning_rate": 0.00016508767113240598,
+      "loss": 3.4830841064453124,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 1.9300287961959839,
+      "learning_rate": 0.00016398281399836097,
+      "loss": 3.309653472900391,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 1.4318655729293823,
+      "learning_rate": 0.00016286457132516383,
+      "loss": 3.589862060546875,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 1.4944521188735962,
+      "learning_rate": 0.0001617331770550274,
+      "loss": 3.277871322631836,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.7870607376098633,
+      "learning_rate": 0.00016058886788154712,
+      "loss": 3.269728088378906,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.5668118000030518,
+      "learning_rate": 0.00015943188320018388,
+      "loss": 3.210224914550781,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 2.267331123352051,
+      "learning_rate": 0.00015826246505818112,
+      "loss": 3.3089523315429688,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 2.414400815963745,
+      "learning_rate": 0.0001570808581039271,
+      "loss": 3.1313955307006838,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 1.8084098100662231,
+      "learning_rate": 0.00015588730953577335,
+      "loss": 3.372232437133789,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 1.4967864751815796,
+      "learning_rate": 0.0001546820690503195,
+      "loss": 3.3817447662353515,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 1.7170981168746948,
+      "learning_rate": 0.0001534653887901754,
+      "loss": 3.1899662017822266,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 1.7449836730957031,
+      "learning_rate": 0.00015223752329121155,
+      "loss": 3.4185081481933595,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 2.0558719635009766,
+      "learning_rate": 0.00015099872942930887,
+      "loss": 3.4337844848632812,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 1.402202844619751,
+      "learning_rate": 0.0001497492663666189,
+      "loss": 3.000233268737793,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 1.575924038887024,
+      "learning_rate": 0.0001484893954973458,
+      "loss": 3.401926803588867,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.8477239608764648,
+      "learning_rate": 0.00014721938039306134,
+      "loss": 3.143557548522949,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.3538575172424316,
+      "learning_rate": 0.00014593948674756417,
+      "loss": 3.316628265380859,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 1.7879822254180908,
+      "learning_rate": 0.0001446499823212954,
+      "loss": 3.032166862487793,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.9181793928146362,
+      "learning_rate": 0.00014335113688532182,
+      "loss": 3.029685401916504,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 1.4723820686340332,
+      "learning_rate": 0.00014204322216489814,
+      "loss": 3.1593692779541014,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 1.5803518295288086,
+      "learning_rate": 0.00014072651178262096,
+      "loss": 3.1910980224609373,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 1.4509774446487427,
+      "learning_rate": 0.00013940128120118516,
+      "loss": 3.3486377716064455,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 1.2490512132644653,
+      "learning_rate": 0.00013806780766575588,
+      "loss": 3.127117919921875,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 1.3815803527832031,
+      "learning_rate": 0.0001367263701459674,
+      "loss": 3.3135940551757814,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 1.6695923805236816,
+      "learning_rate": 0.00013537724927756094,
+      "loss": 3.24918212890625,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 1.6700210571289062,
+      "learning_rate": 0.00013402072730367475,
+      "loss": 3.1998600006103515,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 1.458272099494934,
+      "learning_rate": 0.0001326570880157967,
+      "loss": 3.2691417694091798,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 1.6035007238388062,
+      "learning_rate": 0.00013128661669439416,
+      "loss": 3.191153717041016,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 1.3588290214538574,
+      "learning_rate": 0.00012990960004923154,
+      "loss": 3.177682113647461,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.3084040880203247,
+      "learning_rate": 0.0001285263261593893,
+      "loss": 3.0778690338134767,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.7576879262924194,
+      "learning_rate": 0.000127137084412996,
+      "loss": 3.063307189941406,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 1.61252760887146,
+      "learning_rate": 0.0001257421654466872,
+      "loss": 3.2951290130615236,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.5347871780395508,
+      "learning_rate": 0.0001243418610848024,
+      "loss": 3.1931894302368162,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 2.2056219577789307,
+      "learning_rate": 0.00012293646427833404,
+      "loss": 3.3431358337402344,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 1.6683447360992432,
+      "learning_rate": 0.00012152626904364067,
+      "loss": 3.057468795776367,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 1.455543875694275,
+      "learning_rate": 0.00012011157040093696,
+      "loss": 3.289105987548828,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 1.7347569465637207,
+      "learning_rate": 0.00011869266431257422,
+      "loss": 2.9250898361206055,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 1.3640249967575073,
+      "learning_rate": 0.00011726984762112328,
+      "loss": 3.325461578369141,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 1.4054839611053467,
+      "learning_rate": 0.00011584341798727365,
+      "loss": 3.168132019042969,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 1.2563236951828003,
+      "learning_rate": 0.0001144136738275612,
+      "loss": 3.237269973754883,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 2.893691062927246,
+      "learning_rate": 0.00011298091425193806,
+      "loss": 3.241581344604492,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 1.9952828884124756,
+      "learning_rate": 0.00011154543900119698,
+      "loss": 3.1403961181640625,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 1.573317050933838,
+      "learning_rate": 0.00011010754838426428,
+      "loss": 3.106332778930664,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 2.1935744285583496,
+      "learning_rate": 0.00010866754321537338,
+      "loss": 3.1031381607055666,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 1.5979467630386353,
+      "learning_rate": 0.00010722572475113315,
+      "loss": 3.1226659774780274,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 1.1873749494552612,
+      "learning_rate": 0.00010578239462750321,
+      "loss": 3.1140846252441405,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 1.8321475982666016,
+      "learning_rate": 0.00010433785479669038,
+      "loss": 3.0106977462768554,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 1.3288156986236572,
+      "learning_rate": 0.00010289240746397832,
+      "loss": 2.8980552673339846,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 1.416706919670105,
+      "learning_rate": 0.00010144635502450508,
+      "loss": 3.141121673583984,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 1.9086672067642212,
+      "learning_rate": 0.0001,
+      "loss": 3.282766342163086,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 1.4448398351669312,
+      "learning_rate": 9.855364497549496e-05,
+      "loss": 2.973546600341797,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 1.2777249813079834,
+      "learning_rate": 9.71075925360217e-05,
+      "loss": 3.145435905456543,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 1.276002287864685,
+      "learning_rate": 9.566214520330966e-05,
+      "loss": 3.2174652099609373,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 1.4613112211227417,
+      "learning_rate": 9.421760537249682e-05,
+      "loss": 3.17722110748291,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 1.1228781938552856,
+      "learning_rate": 9.277427524886689e-05,
+      "loss": 2.98958740234375,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 1.1255987882614136,
+      "learning_rate": 9.133245678462663e-05,
+      "loss": 3.1424510955810545,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.194167137145996,
+      "learning_rate": 8.989245161573576e-05,
+      "loss": 3.0195308685302735,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 1.864384412765503,
+      "learning_rate": 8.845456099880303e-05,
+      "loss": 2.997172737121582,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 1.241004467010498,
+      "learning_rate": 8.701908574806197e-05,
+      "loss": 2.9221866607666014,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 1.1507432460784912,
+      "learning_rate": 8.55863261724388e-05,
+      "loss": 2.9324596405029295,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.287448763847351,
+      "learning_rate": 8.415658201272636e-05,
+      "loss": 2.975849914550781,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 1.6067427396774292,
+      "learning_rate": 8.273015237887673e-05,
+      "loss": 2.980916213989258,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.4008774757385254,
+      "learning_rate": 8.130733568742579e-05,
+      "loss": 2.8944910049438475,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 1.2580070495605469,
+      "learning_rate": 7.988842959906306e-05,
+      "loss": 2.969981384277344,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 1.4962762594223022,
+      "learning_rate": 7.847373095635937e-05,
+      "loss": 3.1294715881347654,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 1.2535191774368286,
+      "learning_rate": 7.706353572166597e-05,
+      "loss": 3.066376876831055,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.8347238898277283,
+      "learning_rate": 7.565813891519765e-05,
+      "loss": 3.0157793045043944,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 1.2819799184799194,
+      "learning_rate": 7.425783455331281e-05,
+      "loss": 3.0389671325683594,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 1.1498677730560303,
+      "learning_rate": 7.2862915587004e-05,
+      "loss": 2.986423301696777,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.1965627670288086,
+      "learning_rate": 7.147367384061074e-05,
+      "loss": 2.941807746887207,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 1.8370661735534668,
+      "learning_rate": 7.009039995076844e-05,
+      "loss": 2.658160400390625,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.418321132659912,
+      "learning_rate": 6.871338330560587e-05,
+      "loss": 3.019854736328125,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 1.3217508792877197,
+      "learning_rate": 6.734291198420333e-05,
+      "loss": 2.8131311416625975,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.207661747932434,
+      "learning_rate": 6.597927269632526e-05,
+      "loss": 3.063412666320801,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 1.1648913621902466,
+      "learning_rate": 6.462275072243908e-05,
+      "loss": 3.0537391662597657,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 1.1196516752243042,
+      "learning_rate": 6.327362985403265e-05,
+      "loss": 3.0568498611450194,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 1.1875085830688477,
+      "learning_rate": 6.193219233424414e-05,
+      "loss": 2.8038040161132813,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.5340442657470703,
+      "learning_rate": 6.059871879881487e-05,
+      "loss": 2.922003746032715,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 1.4385885000228882,
+      "learning_rate": 5.927348821737906e-05,
+      "loss": 2.950858306884766,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 1.0835936069488525,
+      "learning_rate": 5.795677783510187e-05,
+      "loss": 2.898311996459961,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 1.4508082866668701,
+      "learning_rate": 5.664886311467821e-05,
+      "loss": 3.036098670959473,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.0718936920166016,
+      "learning_rate": 5.535001767870463e-05,
+      "loss": 2.8239120483398437,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 1.3246824741363525,
+      "learning_rate": 5.406051325243586e-05,
+      "loss": 2.9098350524902346,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.3559950590133667,
+      "learning_rate": 5.278061960693865e-05,
+      "loss": 2.7230411529541017,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 1.3093736171722412,
+      "learning_rate": 5.151060450265419e-05,
+      "loss": 2.998219871520996,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 1.1871122121810913,
+      "learning_rate": 5.025073363338111e-05,
+      "loss": 2.8021818161010743,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.5887531042099,
+      "learning_rate": 4.900127057069116e-05,
+      "loss": 3.138778305053711,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 1.0622819662094116,
+      "learning_rate": 4.776247670878847e-05,
+      "loss": 2.8708589553833006,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.0821974277496338,
+      "learning_rate": 4.65346112098246e-05,
+      "loss": 2.923636054992676,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 1.2138561010360718,
+      "learning_rate": 4.5317930949680524e-05,
+      "loss": 3.0319366455078125,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 1.3472957611083984,
+      "learning_rate": 4.41126904642267e-05,
+      "loss": 2.7567047119140624,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 1.985767126083374,
+      "learning_rate": 4.291914189607297e-05,
+      "loss": 3.056001663208008,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 1.1290528774261475,
+      "learning_rate": 4.17375349418189e-05,
+      "loss": 3.075904655456543,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 1.2921662330627441,
+      "learning_rate": 4.056811679981609e-05,
+      "loss": 2.8820091247558595,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 1.1284035444259644,
+      "learning_rate": 3.9411132118452896e-05,
+      "loss": 2.897430419921875,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1220782995224,
+      "learning_rate": 3.826682294497265e-05,
+      "loss": 2.782516860961914,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 1.0530436038970947,
+      "learning_rate": 3.7135428674836184e-05,
+      "loss": 2.7634408950805662,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 1.3937631845474243,
+      "learning_rate": 3.6017186001639036e-05,
+      "loss": 2.894866180419922,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 1.00092613697052,
+      "learning_rate": 3.491232886759398e-05,
+      "loss": 2.6879959106445312,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 1.1270594596862793,
+      "learning_rate": 3.382108841458904e-05,
+      "loss": 3.027115821838379,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 1.3118301630020142,
+      "learning_rate": 3.274369293583121e-05,
+      "loss": 2.8866352081298827,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 1.2168943881988525,
+      "learning_rate": 3.1680367828086676e-05,
+      "loss": 2.8968345642089846,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 1.0909639596939087,
+      "learning_rate": 3.063133554452645e-05,
+      "loss": 2.9398611068725584,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 1.1608167886734009,
+      "learning_rate": 2.9596815548187908e-05,
+      "loss": 2.842852783203125,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 1.511507511138916,
+      "learning_rate": 2.8577024266062146e-05,
+      "loss": 2.77429141998291,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 1.1318535804748535,
+      "learning_rate": 2.7572175043816206e-05,
+      "loss": 2.8206886291503905,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.9189174771308899,
+      "learning_rate": 2.6582478101160167e-05,
+      "loss": 2.9603801727294923,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.180462121963501,
+      "learning_rate": 2.560814048786799e-05,
+      "loss": 2.8037836074829103,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 1.1114805936813354,
+      "learning_rate": 2.4649366040461597e-05,
+      "loss": 2.941350746154785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 1.31864595413208,
+      "learning_rate": 2.3706355339567286e-05,
+      "loss": 3.0531469345092774,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.9089719653129578,
+      "learning_rate": 2.2779305667953154e-05,
+      "loss": 2.746686553955078,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 1.0223416090011597,
+      "learning_rate": 2.186841096925657e-05,
+      "loss": 2.8307025909423826,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 1.27829909324646,
+      "learning_rate": 2.097386180741019e-05,
+      "loss": 2.928578758239746,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 1.3367505073547363,
+      "learning_rate": 2.0095845326774697e-05,
+      "loss": 2.8606313705444335,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 1.1823525428771973,
+      "learning_rate": 1.9234545212987688e-05,
+      "loss": 3.041524124145508,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 1.2242523431777954,
+      "learning_rate": 1.8390141654535265e-05,
+      "loss": 3.262685775756836,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.9626563191413879,
+      "learning_rate": 1.756281130505595e-05,
+      "loss": 2.824736976623535,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.9038745760917664,
+      "learning_rate": 1.6752727246383582e-05,
+      "loss": 3.0037527084350586,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 1.0726526975631714,
+      "learning_rate": 1.5960058952337887e-05,
+      "loss": 2.818869781494141,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 1.0472805500030518,
+      "learning_rate": 1.518497225326958e-05,
+      "loss": 2.9350055694580077,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 1.1116654872894287,
+      "learning_rate": 1.4427629301367773e-05,
+      "loss": 2.903761291503906,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.9501079320907593,
+      "learning_rate": 1.3688188536736968e-05,
+      "loss": 2.784720611572266,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 1.109456181526184,
+      "learning_rate": 1.2966804654250465e-05,
+      "loss": 3.1309579849243163,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 1.1933435201644897,
+      "learning_rate": 1.2263628571187414e-05,
+      "loss": 3.0474115371704102,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 1.4391895532608032,
+      "learning_rate": 1.1578807395660207e-05,
+      "loss": 3.0081947326660154,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.9554082155227661,
+      "learning_rate": 1.0912484395838585e-05,
+      "loss": 2.9179988861083985,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 1.1937369108200073,
+      "learning_rate": 1.026479896997723e-05,
+      "loss": 2.705255126953125,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.9862006306648254,
+      "learning_rate": 9.635886617252975e-06,
+      "loss": 2.745578384399414,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 1.1251375675201416,
+      "learning_rate": 9.025878909417552e-06,
+      "loss": 2.965798187255859,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 1.0850991010665894,
+      "learning_rate": 8.434903463272215e-06,
+      "loss": 2.9574241638183594,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 1.476346731185913,
+      "learning_rate": 7.86308391396956e-06,
+      "loss": 2.7245121002197266,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 1.181896686553955,
+      "learning_rate": 7.310539889148404e-06,
+      "loss": 3.0273662567138673,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.4082521200180054,
+      "learning_rate": 6.777386983907152e-06,
+      "loss": 2.9661121368408203,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 1.110843539237976,
+      "learning_rate": 6.263736736620551e-06,
+      "loss": 2.999568557739258,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1253182888031006,
+      "learning_rate": 5.769696605605379e-06,
+      "loss": 2.998799705505371,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.089568853378296,
+      "learning_rate": 5.295369946639628e-06,
+      "loss": 2.6438697814941405,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 1.1276156902313232,
+      "learning_rate": 4.840855991339799e-06,
+      "loss": 2.9343143463134767,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 1.0737465620040894,
+      "learning_rate": 4.406249826401309e-06,
+      "loss": 3.080368995666504,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.8834953904151917,
+      "learning_rate": 3.991642373705695e-06,
+      "loss": 2.917354965209961,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 1.0340644121170044,
+      "learning_rate": 3.5971203712993894e-06,
+      "loss": 2.9122617721557615,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 1.1595566272735596,
+      "learning_rate": 3.2227663552476194e-06,
+      "loss": 2.817066764831543,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 1.0316624641418457,
+      "learning_rate": 2.868658642367439e-06,
+      "loss": 2.764511489868164,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 1.1504026651382446,
+      "learning_rate": 2.5348713138434564e-06,
+      "loss": 2.9506311416625977,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 1.0357000827789307,
+      "learning_rate": 2.2214741997296117e-06,
+      "loss": 2.817026901245117,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8134933710098267,
+      "learning_rate": 1.928532864340438e-06,
+      "loss": 2.7651405334472656,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 1.3359462022781372,
+      "learning_rate": 1.6561085925346332e-06,
+      "loss": 2.8869171142578125,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 1.4075947999954224,
+      "learning_rate": 1.4042583768939298e-06,
+      "loss": 3.0473421096801756,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7921971082687378,
+      "learning_rate": 1.1730349057999412e-06,
+      "loss": 2.8001834869384767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 1.0570473670959473,
+      "learning_rate": 9.624865524115346e-07,
+      "loss": 2.9640541076660156,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 1.0891289710998535,
+      "learning_rate": 7.726573645448487e-07,
+      "loss": 3.0531845092773438,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 1.0408469438552856,
+      "learning_rate": 6.03587055458299e-07,
+      "loss": 2.9867210388183594,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.054602861404419,
+      "learning_rate": 4.5531099554435576e-07,
+      "loss": 2.8087078094482423,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 1.1407268047332764,
+      "learning_rate": 3.278602049298418e-07,
+      "loss": 3.1718997955322266,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.9369794130325317,
+      "learning_rate": 2.2126134698637892e-07,
+      "loss": 2.9225208282470705,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 1.6008250713348389,
+      "learning_rate": 1.3553672275230523e-07,
+      "loss": 3.1131832122802736,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.9389436841011047,
+      "learning_rate": 7.070426626711868e-08,
+      "loss": 2.8224658966064453,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.8056095242500305,
+      "learning_rate": 2.677754081961137e-08,
+      "loss": 2.772154998779297,
+      "step": 1200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.518116653061744e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/chat_template.jinja b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..90da567b458e2df298e5ce54139e0704e4b1aa24
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/config.json
@@ -0,0 +1,250 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_rotary_dim": 64,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 1.0,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "partial_rotary_factor": 0.5,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/generation_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/processor_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/tokenizer_config.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/trainer_state.json b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..859f0bae403a62c45aba24612f7bb9dbac20e8ea
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/checkpoint-1207/trainer_state.json
@@ -0,0 +1,1721 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 1147.55224609375,
+      "learning_rate": 6.611570247933885e-06,
+      "loss": 11.37186279296875,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 246.35504150390625,
+      "learning_rate": 1.487603305785124e-05,
+      "loss": 9.2022216796875,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 79.90882110595703,
+      "learning_rate": 2.3140495867768598e-05,
+      "loss": 7.6917167663574215,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 158.79872131347656,
+      "learning_rate": 3.1404958677685955e-05,
+      "loss": 6.138848114013672,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 48.520408630371094,
+      "learning_rate": 3.9669421487603306e-05,
+      "loss": 5.638393402099609,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 14.776691436767578,
+      "learning_rate": 4.793388429752066e-05,
+      "loss": 5.221096801757812,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 12.125936508178711,
+      "learning_rate": 5.619834710743802e-05,
+      "loss": 4.968892288208008,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 14.540782928466797,
+      "learning_rate": 6.446280991735537e-05,
+      "loss": 5.3284343719482425,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 13.903443336486816,
+      "learning_rate": 7.272727272727273e-05,
+      "loss": 5.018967819213867,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 12.485719680786133,
+      "learning_rate": 8.099173553719009e-05,
+      "loss": 4.600848770141601,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 44.14723205566406,
+      "learning_rate": 8.925619834710744e-05,
+      "loss": 4.335272979736328,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 9.01069164276123,
+      "learning_rate": 9.75206611570248e-05,
+      "loss": 4.316203308105469,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 16.169389724731445,
+      "learning_rate": 0.00010578512396694216,
+      "loss": 4.197259140014649,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 6.058357238769531,
+      "learning_rate": 0.0001140495867768595,
+      "loss": 4.562212371826172,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 5.126786231994629,
+      "learning_rate": 0.00012231404958677685,
+      "loss": 4.340535354614258,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 12.424266815185547,
+      "learning_rate": 0.00013057851239669423,
+      "loss": 3.9918304443359376,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 8.042577743530273,
+      "learning_rate": 0.0001388429752066116,
+      "loss": 4.087551879882812,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 8.642739295959473,
+      "learning_rate": 0.00014710743801652894,
+      "loss": 4.22486686706543,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 8.129426002502441,
+      "learning_rate": 0.00015537190082644627,
+      "loss": 3.8094669342041017,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 10.393760681152344,
+      "learning_rate": 0.00016363636363636366,
+      "loss": 3.687621307373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 23.797346115112305,
+      "learning_rate": 0.00017190082644628102,
+      "loss": 4.8999065399169925,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 7.251102924346924,
+      "learning_rate": 0.00018016528925619835,
+      "loss": 4.111215972900391,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 7.632305145263672,
+      "learning_rate": 0.0001884297520661157,
+      "loss": 3.88475341796875,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 2.708984851837158,
+      "learning_rate": 0.0001966942148760331,
+      "loss": 3.9735763549804686,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 3.5484111309051514,
+      "learning_rate": 0.00019999623426388962,
+      "loss": 3.911006546020508,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 3.580739974975586,
+      "learning_rate": 0.0001999732224591804,
+      "loss": 3.6748367309570313,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 3.1602091789245605,
+      "learning_rate": 0.0001999292957337329,
+      "loss": 4.212306213378906,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 2.242072582244873,
+      "learning_rate": 0.0001998644632772477,
+      "loss": 3.8557418823242187,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 3.2909867763519287,
+      "learning_rate": 0.00019977873865301363,
+      "loss": 3.953845977783203,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 2.695601224899292,
+      "learning_rate": 0.00019967213979507018,
+      "loss": 3.7311931610107423,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 3.831533670425415,
+      "learning_rate": 0.00019954468900445566,
+      "loss": 3.88598518371582,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 4.886938571929932,
+      "learning_rate": 0.00019939641294454172,
+      "loss": 3.8442028045654295,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 3.451791524887085,
+      "learning_rate": 0.00019922734263545517,
+      "loss": 3.9103469848632812,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.0034420490264893,
+      "learning_rate": 0.00019903751344758848,
+      "loss": 4.032486343383789,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 2.849339723587036,
+      "learning_rate": 0.00019882696509420007,
+      "loss": 3.407342529296875,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 2.095475435256958,
+      "learning_rate": 0.00019859574162310608,
+      "loss": 3.7116188049316405,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 3.148158073425293,
+      "learning_rate": 0.0001983438914074654,
+      "loss": 3.7624229431152343,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 2.4163601398468018,
+      "learning_rate": 0.00019807146713565955,
+      "loss": 3.439311218261719,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 2.7623701095581055,
+      "learning_rate": 0.00019777852580027042,
+      "loss": 3.8211971282958985,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 3.145005226135254,
+      "learning_rate": 0.00019746512868615656,
+      "loss": 3.5431480407714844,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 2.633315324783325,
+      "learning_rate": 0.00019713134135763256,
+      "loss": 3.784006881713867,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 2.0188381671905518,
+      "learning_rate": 0.00019677723364475237,
+      "loss": 3.7024520874023437,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 2.1343772411346436,
+      "learning_rate": 0.00019640287962870062,
+      "loss": 3.7561168670654297,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 1.9709585905075073,
+      "learning_rate": 0.0001960083576262943,
+      "loss": 3.6580154418945314,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 2.0804245471954346,
+      "learning_rate": 0.0001955937501735987,
+      "loss": 3.379087448120117,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 1.890079379081726,
+      "learning_rate": 0.0001951591440086602,
+      "loss": 3.5966243743896484,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 2.0973691940307617,
+      "learning_rate": 0.0001947046300533604,
+      "loss": 3.4583953857421874,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 2.4010658264160156,
+      "learning_rate": 0.00019423030339439464,
+      "loss": 3.3876201629638674,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 2.172189235687256,
+      "learning_rate": 0.00019373626326337946,
+      "loss": 3.7462398529052736,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 1.9334107637405396,
+      "learning_rate": 0.00019322261301609286,
+      "loss": 3.841419219970703,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 2.0680906772613525,
+      "learning_rate": 0.00019268946011085158,
+      "loss": 3.4502696990966797,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 2.5843758583068848,
+      "learning_rate": 0.00019213691608603047,
+      "loss": 3.404983139038086,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 1.888687252998352,
+      "learning_rate": 0.00019156509653672782,
+      "loss": 3.549253463745117,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 1.8593695163726807,
+      "learning_rate": 0.00019097412109058247,
+      "loss": 3.422663116455078,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 2.036330461502075,
+      "learning_rate": 0.00019036411338274703,
+      "loss": 3.7582332611083986,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 1.8483343124389648,
+      "learning_rate": 0.00018973520103002277,
+      "loss": 3.517898941040039,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 2.0232744216918945,
+      "learning_rate": 0.00018908751560416142,
+      "loss": 3.521917724609375,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 3.3193116188049316,
+      "learning_rate": 0.00018842119260433982,
+      "loss": 3.4880836486816404,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 2.5043585300445557,
+      "learning_rate": 0.0001877363714288126,
+      "loss": 3.446257400512695,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 1.9308786392211914,
+      "learning_rate": 0.00018703319534574956,
+      "loss": 3.3181568145751954,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 1.857397198677063,
+      "learning_rate": 0.00018631181146326305,
+      "loss": 3.219373321533203,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.9737612009048462,
+      "learning_rate": 0.00018557237069863222,
+      "loss": 3.5038528442382812,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 4.4995598793029785,
+      "learning_rate": 0.00018481502774673044,
+      "loss": 3.3514575958251953,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 2.948864221572876,
+      "learning_rate": 0.00018403994104766212,
+      "loss": 3.13072509765625,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 2.1394331455230713,
+      "learning_rate": 0.00018324727275361645,
+      "loss": 3.5682483673095704,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 3.813765287399292,
+      "learning_rate": 0.00018243718869494408,
+      "loss": 3.478915786743164,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.6351795196533203,
+      "learning_rate": 0.00018160985834546475,
+      "loss": 3.307242584228516,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 2.2016730308532715,
+      "learning_rate": 0.00018076545478701235,
+      "loss": 3.5482852935791014,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 2.1184046268463135,
+      "learning_rate": 0.00017990415467322534,
+      "loss": 3.51483039855957,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 1.704121470451355,
+      "learning_rate": 0.00017902613819258985,
+      "loss": 3.6264686584472656,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 1.926048994064331,
+      "learning_rate": 0.00017813158903074342,
+      "loss": 3.472669219970703,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 2.019219160079956,
+      "learning_rate": 0.00017722069433204687,
+      "loss": 3.2899856567382812,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 2.0369884967803955,
+      "learning_rate": 0.00017629364466043273,
+      "loss": 3.427472686767578,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 1.7189043760299683,
+      "learning_rate": 0.0001753506339595384,
+      "loss": 3.403704833984375,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 2.494591236114502,
+      "learning_rate": 0.00017439185951213202,
+      "loss": 3.5159603118896485,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 1.7678285837173462,
+      "learning_rate": 0.00017341752189883983,
+      "loss": 3.119024467468262,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 2.24629807472229,
+      "learning_rate": 0.0001724278249561838,
+      "loss": 3.525630569458008,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 1.8001821041107178,
+      "learning_rate": 0.0001714229757339379,
+      "loss": 3.541637420654297,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 2.8331096172332764,
+      "learning_rate": 0.0001704031844518121,
+      "loss": 3.3902435302734375,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 2.052628755569458,
+      "learning_rate": 0.00016936866445547353,
+      "loss": 3.218609619140625,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 2.2566065788269043,
+      "learning_rate": 0.00016831963217191334,
+      "loss": 3.5105533599853516,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.5952314138412476,
+      "learning_rate": 0.0001672563070641688,
+      "loss": 3.253592681884766,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 1.6297531127929688,
+      "learning_rate": 0.000166178911585411,
+      "loss": 3.176485061645508,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 1.6768420934677124,
+      "learning_rate": 0.00016508767113240598,
+      "loss": 3.4830841064453124,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 1.9300287961959839,
+      "learning_rate": 0.00016398281399836097,
+      "loss": 3.309653472900391,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 1.4318655729293823,
+      "learning_rate": 0.00016286457132516383,
+      "loss": 3.589862060546875,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 1.4944521188735962,
+      "learning_rate": 0.0001617331770550274,
+      "loss": 3.277871322631836,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.7870607376098633,
+      "learning_rate": 0.00016058886788154712,
+      "loss": 3.269728088378906,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.5668118000030518,
+      "learning_rate": 0.00015943188320018388,
+      "loss": 3.210224914550781,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 2.267331123352051,
+      "learning_rate": 0.00015826246505818112,
+      "loss": 3.3089523315429688,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 2.414400815963745,
+      "learning_rate": 0.0001570808581039271,
+      "loss": 3.1313955307006838,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 1.8084098100662231,
+      "learning_rate": 0.00015588730953577335,
+      "loss": 3.372232437133789,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 1.4967864751815796,
+      "learning_rate": 0.0001546820690503195,
+      "loss": 3.3817447662353515,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 1.7170981168746948,
+      "learning_rate": 0.0001534653887901754,
+      "loss": 3.1899662017822266,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 1.7449836730957031,
+      "learning_rate": 0.00015223752329121155,
+      "loss": 3.4185081481933595,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 2.0558719635009766,
+      "learning_rate": 0.00015099872942930887,
+      "loss": 3.4337844848632812,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 1.402202844619751,
+      "learning_rate": 0.0001497492663666189,
+      "loss": 3.000233268737793,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 1.575924038887024,
+      "learning_rate": 0.0001484893954973458,
+      "loss": 3.401926803588867,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.8477239608764648,
+      "learning_rate": 0.00014721938039306134,
+      "loss": 3.143557548522949,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.3538575172424316,
+      "learning_rate": 0.00014593948674756417,
+      "loss": 3.316628265380859,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 1.7879822254180908,
+      "learning_rate": 0.0001446499823212954,
+      "loss": 3.032166862487793,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.9181793928146362,
+      "learning_rate": 0.00014335113688532182,
+      "loss": 3.029685401916504,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 1.4723820686340332,
+      "learning_rate": 0.00014204322216489814,
+      "loss": 3.1593692779541014,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 1.5803518295288086,
+      "learning_rate": 0.00014072651178262096,
+      "loss": 3.1910980224609373,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 1.4509774446487427,
+      "learning_rate": 0.00013940128120118516,
+      "loss": 3.3486377716064455,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 1.2490512132644653,
+      "learning_rate": 0.00013806780766575588,
+      "loss": 3.127117919921875,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 1.3815803527832031,
+      "learning_rate": 0.0001367263701459674,
+      "loss": 3.3135940551757814,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 1.6695923805236816,
+      "learning_rate": 0.00013537724927756094,
+      "loss": 3.24918212890625,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 1.6700210571289062,
+      "learning_rate": 0.00013402072730367475,
+      "loss": 3.1998600006103515,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 1.458272099494934,
+      "learning_rate": 0.0001326570880157967,
+      "loss": 3.2691417694091798,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 1.6035007238388062,
+      "learning_rate": 0.00013128661669439416,
+      "loss": 3.191153717041016,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 1.3588290214538574,
+      "learning_rate": 0.00012990960004923154,
+      "loss": 3.177682113647461,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.3084040880203247,
+      "learning_rate": 0.0001285263261593893,
+      "loss": 3.0778690338134767,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.7576879262924194,
+      "learning_rate": 0.000127137084412996,
+      "loss": 3.063307189941406,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 1.61252760887146,
+      "learning_rate": 0.0001257421654466872,
+      "loss": 3.2951290130615236,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.5347871780395508,
+      "learning_rate": 0.0001243418610848024,
+      "loss": 3.1931894302368162,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 2.2056219577789307,
+      "learning_rate": 0.00012293646427833404,
+      "loss": 3.3431358337402344,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 1.6683447360992432,
+      "learning_rate": 0.00012152626904364067,
+      "loss": 3.057468795776367,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 1.455543875694275,
+      "learning_rate": 0.00012011157040093696,
+      "loss": 3.289105987548828,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 1.7347569465637207,
+      "learning_rate": 0.00011869266431257422,
+      "loss": 2.9250898361206055,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 1.3640249967575073,
+      "learning_rate": 0.00011726984762112328,
+      "loss": 3.325461578369141,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 1.4054839611053467,
+      "learning_rate": 0.00011584341798727365,
+      "loss": 3.168132019042969,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 1.2563236951828003,
+      "learning_rate": 0.0001144136738275612,
+      "loss": 3.237269973754883,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 2.893691062927246,
+      "learning_rate": 0.00011298091425193806,
+      "loss": 3.241581344604492,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 1.9952828884124756,
+      "learning_rate": 0.00011154543900119698,
+      "loss": 3.1403961181640625,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 1.573317050933838,
+      "learning_rate": 0.00011010754838426428,
+      "loss": 3.106332778930664,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 2.1935744285583496,
+      "learning_rate": 0.00010866754321537338,
+      "loss": 3.1031381607055666,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 1.5979467630386353,
+      "learning_rate": 0.00010722572475113315,
+      "loss": 3.1226659774780274,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 1.1873749494552612,
+      "learning_rate": 0.00010578239462750321,
+      "loss": 3.1140846252441405,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 1.8321475982666016,
+      "learning_rate": 0.00010433785479669038,
+      "loss": 3.0106977462768554,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 1.3288156986236572,
+      "learning_rate": 0.00010289240746397832,
+      "loss": 2.8980552673339846,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 1.416706919670105,
+      "learning_rate": 0.00010144635502450508,
+      "loss": 3.141121673583984,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 1.9086672067642212,
+      "learning_rate": 0.0001,
+      "loss": 3.282766342163086,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 1.4448398351669312,
+      "learning_rate": 9.855364497549496e-05,
+      "loss": 2.973546600341797,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 1.2777249813079834,
+      "learning_rate": 9.71075925360217e-05,
+      "loss": 3.145435905456543,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 1.276002287864685,
+      "learning_rate": 9.566214520330966e-05,
+      "loss": 3.2174652099609373,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 1.4613112211227417,
+      "learning_rate": 9.421760537249682e-05,
+      "loss": 3.17722110748291,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 1.1228781938552856,
+      "learning_rate": 9.277427524886689e-05,
+      "loss": 2.98958740234375,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 1.1255987882614136,
+      "learning_rate": 9.133245678462663e-05,
+      "loss": 3.1424510955810545,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.194167137145996,
+      "learning_rate": 8.989245161573576e-05,
+      "loss": 3.0195308685302735,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 1.864384412765503,
+      "learning_rate": 8.845456099880303e-05,
+      "loss": 2.997172737121582,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 1.241004467010498,
+      "learning_rate": 8.701908574806197e-05,
+      "loss": 2.9221866607666014,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 1.1507432460784912,
+      "learning_rate": 8.55863261724388e-05,
+      "loss": 2.9324596405029295,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.287448763847351,
+      "learning_rate": 8.415658201272636e-05,
+      "loss": 2.975849914550781,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 1.6067427396774292,
+      "learning_rate": 8.273015237887673e-05,
+      "loss": 2.980916213989258,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.4008774757385254,
+      "learning_rate": 8.130733568742579e-05,
+      "loss": 2.8944910049438475,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 1.2580070495605469,
+      "learning_rate": 7.988842959906306e-05,
+      "loss": 2.969981384277344,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 1.4962762594223022,
+      "learning_rate": 7.847373095635937e-05,
+      "loss": 3.1294715881347654,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 1.2535191774368286,
+      "learning_rate": 7.706353572166597e-05,
+      "loss": 3.066376876831055,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.8347238898277283,
+      "learning_rate": 7.565813891519765e-05,
+      "loss": 3.0157793045043944,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 1.2819799184799194,
+      "learning_rate": 7.425783455331281e-05,
+      "loss": 3.0389671325683594,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 1.1498677730560303,
+      "learning_rate": 7.2862915587004e-05,
+      "loss": 2.986423301696777,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.1965627670288086,
+      "learning_rate": 7.147367384061074e-05,
+      "loss": 2.941807746887207,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 1.8370661735534668,
+      "learning_rate": 7.009039995076844e-05,
+      "loss": 2.658160400390625,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.418321132659912,
+      "learning_rate": 6.871338330560587e-05,
+      "loss": 3.019854736328125,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 1.3217508792877197,
+      "learning_rate": 6.734291198420333e-05,
+      "loss": 2.8131311416625975,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.207661747932434,
+      "learning_rate": 6.597927269632526e-05,
+      "loss": 3.063412666320801,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 1.1648913621902466,
+      "learning_rate": 6.462275072243908e-05,
+      "loss": 3.0537391662597657,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 1.1196516752243042,
+      "learning_rate": 6.327362985403265e-05,
+      "loss": 3.0568498611450194,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 1.1875085830688477,
+      "learning_rate": 6.193219233424414e-05,
+      "loss": 2.8038040161132813,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.5340442657470703,
+      "learning_rate": 6.059871879881487e-05,
+      "loss": 2.922003746032715,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 1.4385885000228882,
+      "learning_rate": 5.927348821737906e-05,
+      "loss": 2.950858306884766,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 1.0835936069488525,
+      "learning_rate": 5.795677783510187e-05,
+      "loss": 2.898311996459961,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 1.4508082866668701,
+      "learning_rate": 5.664886311467821e-05,
+      "loss": 3.036098670959473,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.0718936920166016,
+      "learning_rate": 5.535001767870463e-05,
+      "loss": 2.8239120483398437,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 1.3246824741363525,
+      "learning_rate": 5.406051325243586e-05,
+      "loss": 2.9098350524902346,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.3559950590133667,
+      "learning_rate": 5.278061960693865e-05,
+      "loss": 2.7230411529541017,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 1.3093736171722412,
+      "learning_rate": 5.151060450265419e-05,
+      "loss": 2.998219871520996,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 1.1871122121810913,
+      "learning_rate": 5.025073363338111e-05,
+      "loss": 2.8021818161010743,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.5887531042099,
+      "learning_rate": 4.900127057069116e-05,
+      "loss": 3.138778305053711,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 1.0622819662094116,
+      "learning_rate": 4.776247670878847e-05,
+      "loss": 2.8708589553833006,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.0821974277496338,
+      "learning_rate": 4.65346112098246e-05,
+      "loss": 2.923636054992676,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 1.2138561010360718,
+      "learning_rate": 4.5317930949680524e-05,
+      "loss": 3.0319366455078125,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 1.3472957611083984,
+      "learning_rate": 4.41126904642267e-05,
+      "loss": 2.7567047119140624,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 1.985767126083374,
+      "learning_rate": 4.291914189607297e-05,
+      "loss": 3.056001663208008,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 1.1290528774261475,
+      "learning_rate": 4.17375349418189e-05,
+      "loss": 3.075904655456543,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 1.2921662330627441,
+      "learning_rate": 4.056811679981609e-05,
+      "loss": 2.8820091247558595,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 1.1284035444259644,
+      "learning_rate": 3.9411132118452896e-05,
+      "loss": 2.897430419921875,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1220782995224,
+      "learning_rate": 3.826682294497265e-05,
+      "loss": 2.782516860961914,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 1.0530436038970947,
+      "learning_rate": 3.7135428674836184e-05,
+      "loss": 2.7634408950805662,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 1.3937631845474243,
+      "learning_rate": 3.6017186001639036e-05,
+      "loss": 2.894866180419922,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 1.00092613697052,
+      "learning_rate": 3.491232886759398e-05,
+      "loss": 2.6879959106445312,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 1.1270594596862793,
+      "learning_rate": 3.382108841458904e-05,
+      "loss": 3.027115821838379,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 1.3118301630020142,
+      "learning_rate": 3.274369293583121e-05,
+      "loss": 2.8866352081298827,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 1.2168943881988525,
+      "learning_rate": 3.1680367828086676e-05,
+      "loss": 2.8968345642089846,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 1.0909639596939087,
+      "learning_rate": 3.063133554452645e-05,
+      "loss": 2.9398611068725584,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 1.1608167886734009,
+      "learning_rate": 2.9596815548187908e-05,
+      "loss": 2.842852783203125,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 1.511507511138916,
+      "learning_rate": 2.8577024266062146e-05,
+      "loss": 2.77429141998291,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 1.1318535804748535,
+      "learning_rate": 2.7572175043816206e-05,
+      "loss": 2.8206886291503905,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.9189174771308899,
+      "learning_rate": 2.6582478101160167e-05,
+      "loss": 2.9603801727294923,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.180462121963501,
+      "learning_rate": 2.560814048786799e-05,
+      "loss": 2.8037836074829103,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 1.1114805936813354,
+      "learning_rate": 2.4649366040461597e-05,
+      "loss": 2.941350746154785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 1.31864595413208,
+      "learning_rate": 2.3706355339567286e-05,
+      "loss": 3.0531469345092774,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.9089719653129578,
+      "learning_rate": 2.2779305667953154e-05,
+      "loss": 2.746686553955078,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 1.0223416090011597,
+      "learning_rate": 2.186841096925657e-05,
+      "loss": 2.8307025909423826,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 1.27829909324646,
+      "learning_rate": 2.097386180741019e-05,
+      "loss": 2.928578758239746,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 1.3367505073547363,
+      "learning_rate": 2.0095845326774697e-05,
+      "loss": 2.8606313705444335,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 1.1823525428771973,
+      "learning_rate": 1.9234545212987688e-05,
+      "loss": 3.041524124145508,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 1.2242523431777954,
+      "learning_rate": 1.8390141654535265e-05,
+      "loss": 3.262685775756836,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.9626563191413879,
+      "learning_rate": 1.756281130505595e-05,
+      "loss": 2.824736976623535,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.9038745760917664,
+      "learning_rate": 1.6752727246383582e-05,
+      "loss": 3.0037527084350586,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 1.0726526975631714,
+      "learning_rate": 1.5960058952337887e-05,
+      "loss": 2.818869781494141,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 1.0472805500030518,
+      "learning_rate": 1.518497225326958e-05,
+      "loss": 2.9350055694580077,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 1.1116654872894287,
+      "learning_rate": 1.4427629301367773e-05,
+      "loss": 2.903761291503906,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.9501079320907593,
+      "learning_rate": 1.3688188536736968e-05,
+      "loss": 2.784720611572266,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 1.109456181526184,
+      "learning_rate": 1.2966804654250465e-05,
+      "loss": 3.1309579849243163,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 1.1933435201644897,
+      "learning_rate": 1.2263628571187414e-05,
+      "loss": 3.0474115371704102,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 1.4391895532608032,
+      "learning_rate": 1.1578807395660207e-05,
+      "loss": 3.0081947326660154,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.9554082155227661,
+      "learning_rate": 1.0912484395838585e-05,
+      "loss": 2.9179988861083985,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 1.1937369108200073,
+      "learning_rate": 1.026479896997723e-05,
+      "loss": 2.705255126953125,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.9862006306648254,
+      "learning_rate": 9.635886617252975e-06,
+      "loss": 2.745578384399414,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 1.1251375675201416,
+      "learning_rate": 9.025878909417552e-06,
+      "loss": 2.965798187255859,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 1.0850991010665894,
+      "learning_rate": 8.434903463272215e-06,
+      "loss": 2.9574241638183594,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 1.476346731185913,
+      "learning_rate": 7.86308391396956e-06,
+      "loss": 2.7245121002197266,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 1.181896686553955,
+      "learning_rate": 7.310539889148404e-06,
+      "loss": 3.0273662567138673,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.4082521200180054,
+      "learning_rate": 6.777386983907152e-06,
+      "loss": 2.9661121368408203,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 1.110843539237976,
+      "learning_rate": 6.263736736620551e-06,
+      "loss": 2.999568557739258,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1253182888031006,
+      "learning_rate": 5.769696605605379e-06,
+      "loss": 2.998799705505371,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.089568853378296,
+      "learning_rate": 5.295369946639628e-06,
+      "loss": 2.6438697814941405,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 1.1276156902313232,
+      "learning_rate": 4.840855991339799e-06,
+      "loss": 2.9343143463134767,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 1.0737465620040894,
+      "learning_rate": 4.406249826401309e-06,
+      "loss": 3.080368995666504,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.8834953904151917,
+      "learning_rate": 3.991642373705695e-06,
+      "loss": 2.917354965209961,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 1.0340644121170044,
+      "learning_rate": 3.5971203712993894e-06,
+      "loss": 2.9122617721557615,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 1.1595566272735596,
+      "learning_rate": 3.2227663552476194e-06,
+      "loss": 2.817066764831543,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 1.0316624641418457,
+      "learning_rate": 2.868658642367439e-06,
+      "loss": 2.764511489868164,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 1.1504026651382446,
+      "learning_rate": 2.5348713138434564e-06,
+      "loss": 2.9506311416625977,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 1.0357000827789307,
+      "learning_rate": 2.2214741997296117e-06,
+      "loss": 2.817026901245117,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8134933710098267,
+      "learning_rate": 1.928532864340438e-06,
+      "loss": 2.7651405334472656,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 1.3359462022781372,
+      "learning_rate": 1.6561085925346332e-06,
+      "loss": 2.8869171142578125,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 1.4075947999954224,
+      "learning_rate": 1.4042583768939298e-06,
+      "loss": 3.0473421096801756,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7921971082687378,
+      "learning_rate": 1.1730349057999412e-06,
+      "loss": 2.8001834869384767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 1.0570473670959473,
+      "learning_rate": 9.624865524115346e-07,
+      "loss": 2.9640541076660156,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 1.0891289710998535,
+      "learning_rate": 7.726573645448487e-07,
+      "loss": 3.0531845092773438,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 1.0408469438552856,
+      "learning_rate": 6.03587055458299e-07,
+      "loss": 2.9867210388183594,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.054602861404419,
+      "learning_rate": 4.5531099554435576e-07,
+      "loss": 2.8087078094482423,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 1.1407268047332764,
+      "learning_rate": 3.278602049298418e-07,
+      "loss": 3.1718997955322266,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.9369794130325317,
+      "learning_rate": 2.2126134698637892e-07,
+      "loss": 2.9225208282470705,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 1.6008250713348389,
+      "learning_rate": 1.3553672275230523e-07,
+      "loss": 3.1131832122802736,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.9389436841011047,
+      "learning_rate": 7.070426626711868e-08,
+      "loss": 2.8224658966064453,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.8056095242500305,
+      "learning_rate": 2.677754081961137e-08,
+      "loss": 2.772154998779297,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.3219596147537231,
+      "learning_rate": 3.7657361103837776e-09,
+      "loss": 3.1340938568115235,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532249769992768e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/config.json b/MOSS_Speaker-RoPE/output_lr2e4/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..90da567b458e2df298e5ce54139e0704e4b1aa24
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/config.json
@@ -0,0 +1,250 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_rotary_dim": 64,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 1.0,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "partial_rotary_factor": 0.5,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/generation_config.json b/MOSS_Speaker-RoPE/output_lr2e4/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/model.safetensors b/MOSS_Speaker-RoPE/output_lr2e4/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a478521340980a738ed98e5c5f90b4f2eee448e5
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac31de076c8011f5f18fb1486b6c05829615dafc57c289a656f1384146d966ed
+size 1867849776
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/processor_config.json b/MOSS_Speaker-RoPE/output_lr2e4/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/tokenizer.json b/MOSS_Speaker-RoPE/output_lr2e4/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f3fe62e4fcd17442942325dec95aa58e731e4a7
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb21895752d2380b32ae2e2c40905c7aa6c8e55c5e6ba33cd065bf77d4d53f74
+size 11423344
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/tokenizer_config.json b/MOSS_Speaker-RoPE/output_lr2e4/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/train_results.json b/MOSS_Speaker-RoPE/output_lr2e4/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1dd88dfc1c5e66693d935f7a6a47d47c7785815
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.532249769992768e+17,
+    "train_loss": 3.365502968245335,
+    "train_runtime": 20256.5023,
+    "train_samples_per_second": 0.06,
+    "train_steps_per_second": 0.06
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/trainer_state.json b/MOSS_Speaker-RoPE/output_lr2e4/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..79d0dda458809b70ffd94cc5d7019054c677260e
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/trainer_state.json
@@ -0,0 +1,1730 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 1147.55224609375,
+      "learning_rate": 6.611570247933885e-06,
+      "loss": 11.37186279296875,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 246.35504150390625,
+      "learning_rate": 1.487603305785124e-05,
+      "loss": 9.2022216796875,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 79.90882110595703,
+      "learning_rate": 2.3140495867768598e-05,
+      "loss": 7.6917167663574215,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 158.79872131347656,
+      "learning_rate": 3.1404958677685955e-05,
+      "loss": 6.138848114013672,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 48.520408630371094,
+      "learning_rate": 3.9669421487603306e-05,
+      "loss": 5.638393402099609,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 14.776691436767578,
+      "learning_rate": 4.793388429752066e-05,
+      "loss": 5.221096801757812,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 12.125936508178711,
+      "learning_rate": 5.619834710743802e-05,
+      "loss": 4.968892288208008,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 14.540782928466797,
+      "learning_rate": 6.446280991735537e-05,
+      "loss": 5.3284343719482425,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 13.903443336486816,
+      "learning_rate": 7.272727272727273e-05,
+      "loss": 5.018967819213867,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 12.485719680786133,
+      "learning_rate": 8.099173553719009e-05,
+      "loss": 4.600848770141601,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 44.14723205566406,
+      "learning_rate": 8.925619834710744e-05,
+      "loss": 4.335272979736328,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 9.01069164276123,
+      "learning_rate": 9.75206611570248e-05,
+      "loss": 4.316203308105469,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 16.169389724731445,
+      "learning_rate": 0.00010578512396694216,
+      "loss": 4.197259140014649,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 6.058357238769531,
+      "learning_rate": 0.0001140495867768595,
+      "loss": 4.562212371826172,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 5.126786231994629,
+      "learning_rate": 0.00012231404958677685,
+      "loss": 4.340535354614258,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 12.424266815185547,
+      "learning_rate": 0.00013057851239669423,
+      "loss": 3.9918304443359376,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 8.042577743530273,
+      "learning_rate": 0.0001388429752066116,
+      "loss": 4.087551879882812,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 8.642739295959473,
+      "learning_rate": 0.00014710743801652894,
+      "loss": 4.22486686706543,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 8.129426002502441,
+      "learning_rate": 0.00015537190082644627,
+      "loss": 3.8094669342041017,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 10.393760681152344,
+      "learning_rate": 0.00016363636363636366,
+      "loss": 3.687621307373047,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 23.797346115112305,
+      "learning_rate": 0.00017190082644628102,
+      "loss": 4.8999065399169925,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 7.251102924346924,
+      "learning_rate": 0.00018016528925619835,
+      "loss": 4.111215972900391,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 7.632305145263672,
+      "learning_rate": 0.0001884297520661157,
+      "loss": 3.88475341796875,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 2.708984851837158,
+      "learning_rate": 0.0001966942148760331,
+      "loss": 3.9735763549804686,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 3.5484111309051514,
+      "learning_rate": 0.00019999623426388962,
+      "loss": 3.911006546020508,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 3.580739974975586,
+      "learning_rate": 0.0001999732224591804,
+      "loss": 3.6748367309570313,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 3.1602091789245605,
+      "learning_rate": 0.0001999292957337329,
+      "loss": 4.212306213378906,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 2.242072582244873,
+      "learning_rate": 0.0001998644632772477,
+      "loss": 3.8557418823242187,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 3.2909867763519287,
+      "learning_rate": 0.00019977873865301363,
+      "loss": 3.953845977783203,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 2.695601224899292,
+      "learning_rate": 0.00019967213979507018,
+      "loss": 3.7311931610107423,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 3.831533670425415,
+      "learning_rate": 0.00019954468900445566,
+      "loss": 3.88598518371582,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 4.886938571929932,
+      "learning_rate": 0.00019939641294454172,
+      "loss": 3.8442028045654295,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 3.451791524887085,
+      "learning_rate": 0.00019922734263545517,
+      "loss": 3.9103469848632812,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.0034420490264893,
+      "learning_rate": 0.00019903751344758848,
+      "loss": 4.032486343383789,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 2.849339723587036,
+      "learning_rate": 0.00019882696509420007,
+      "loss": 3.407342529296875,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 2.095475435256958,
+      "learning_rate": 0.00019859574162310608,
+      "loss": 3.7116188049316405,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 3.148158073425293,
+      "learning_rate": 0.0001983438914074654,
+      "loss": 3.7624229431152343,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 2.4163601398468018,
+      "learning_rate": 0.00019807146713565955,
+      "loss": 3.439311218261719,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 2.7623701095581055,
+      "learning_rate": 0.00019777852580027042,
+      "loss": 3.8211971282958985,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 3.145005226135254,
+      "learning_rate": 0.00019746512868615656,
+      "loss": 3.5431480407714844,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 2.633315324783325,
+      "learning_rate": 0.00019713134135763256,
+      "loss": 3.784006881713867,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 2.0188381671905518,
+      "learning_rate": 0.00019677723364475237,
+      "loss": 3.7024520874023437,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 2.1343772411346436,
+      "learning_rate": 0.00019640287962870062,
+      "loss": 3.7561168670654297,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 1.9709585905075073,
+      "learning_rate": 0.0001960083576262943,
+      "loss": 3.6580154418945314,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 2.0804245471954346,
+      "learning_rate": 0.0001955937501735987,
+      "loss": 3.379087448120117,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 1.890079379081726,
+      "learning_rate": 0.0001951591440086602,
+      "loss": 3.5966243743896484,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 2.0973691940307617,
+      "learning_rate": 0.0001947046300533604,
+      "loss": 3.4583953857421874,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 2.4010658264160156,
+      "learning_rate": 0.00019423030339439464,
+      "loss": 3.3876201629638674,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 2.172189235687256,
+      "learning_rate": 0.00019373626326337946,
+      "loss": 3.7462398529052736,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 1.9334107637405396,
+      "learning_rate": 0.00019322261301609286,
+      "loss": 3.841419219970703,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 2.0680906772613525,
+      "learning_rate": 0.00019268946011085158,
+      "loss": 3.4502696990966797,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 2.5843758583068848,
+      "learning_rate": 0.00019213691608603047,
+      "loss": 3.404983139038086,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 1.888687252998352,
+      "learning_rate": 0.00019156509653672782,
+      "loss": 3.549253463745117,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 1.8593695163726807,
+      "learning_rate": 0.00019097412109058247,
+      "loss": 3.422663116455078,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 2.036330461502075,
+      "learning_rate": 0.00019036411338274703,
+      "loss": 3.7582332611083986,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 1.8483343124389648,
+      "learning_rate": 0.00018973520103002277,
+      "loss": 3.517898941040039,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 2.0232744216918945,
+      "learning_rate": 0.00018908751560416142,
+      "loss": 3.521917724609375,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 3.3193116188049316,
+      "learning_rate": 0.00018842119260433982,
+      "loss": 3.4880836486816404,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 2.5043585300445557,
+      "learning_rate": 0.0001877363714288126,
+      "loss": 3.446257400512695,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 1.9308786392211914,
+      "learning_rate": 0.00018703319534574956,
+      "loss": 3.3181568145751954,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 1.857397198677063,
+      "learning_rate": 0.00018631181146326305,
+      "loss": 3.219373321533203,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.9737612009048462,
+      "learning_rate": 0.00018557237069863222,
+      "loss": 3.5038528442382812,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 4.4995598793029785,
+      "learning_rate": 0.00018481502774673044,
+      "loss": 3.3514575958251953,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 2.948864221572876,
+      "learning_rate": 0.00018403994104766212,
+      "loss": 3.13072509765625,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 2.1394331455230713,
+      "learning_rate": 0.00018324727275361645,
+      "loss": 3.5682483673095704,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 3.813765287399292,
+      "learning_rate": 0.00018243718869494408,
+      "loss": 3.478915786743164,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.6351795196533203,
+      "learning_rate": 0.00018160985834546475,
+      "loss": 3.307242584228516,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 2.2016730308532715,
+      "learning_rate": 0.00018076545478701235,
+      "loss": 3.5482852935791014,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 2.1184046268463135,
+      "learning_rate": 0.00017990415467322534,
+      "loss": 3.51483039855957,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 1.704121470451355,
+      "learning_rate": 0.00017902613819258985,
+      "loss": 3.6264686584472656,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 1.926048994064331,
+      "learning_rate": 0.00017813158903074342,
+      "loss": 3.472669219970703,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 2.019219160079956,
+      "learning_rate": 0.00017722069433204687,
+      "loss": 3.2899856567382812,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 2.0369884967803955,
+      "learning_rate": 0.00017629364466043273,
+      "loss": 3.427472686767578,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 1.7189043760299683,
+      "learning_rate": 0.0001753506339595384,
+      "loss": 3.403704833984375,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 2.494591236114502,
+      "learning_rate": 0.00017439185951213202,
+      "loss": 3.5159603118896485,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 1.7678285837173462,
+      "learning_rate": 0.00017341752189883983,
+      "loss": 3.119024467468262,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 2.24629807472229,
+      "learning_rate": 0.0001724278249561838,
+      "loss": 3.525630569458008,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 1.8001821041107178,
+      "learning_rate": 0.0001714229757339379,
+      "loss": 3.541637420654297,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 2.8331096172332764,
+      "learning_rate": 0.0001704031844518121,
+      "loss": 3.3902435302734375,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 2.052628755569458,
+      "learning_rate": 0.00016936866445547353,
+      "loss": 3.218609619140625,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 2.2566065788269043,
+      "learning_rate": 0.00016831963217191334,
+      "loss": 3.5105533599853516,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.5952314138412476,
+      "learning_rate": 0.0001672563070641688,
+      "loss": 3.253592681884766,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 1.6297531127929688,
+      "learning_rate": 0.000166178911585411,
+      "loss": 3.176485061645508,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 1.6768420934677124,
+      "learning_rate": 0.00016508767113240598,
+      "loss": 3.4830841064453124,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 1.9300287961959839,
+      "learning_rate": 0.00016398281399836097,
+      "loss": 3.309653472900391,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 1.4318655729293823,
+      "learning_rate": 0.00016286457132516383,
+      "loss": 3.589862060546875,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 1.4944521188735962,
+      "learning_rate": 0.0001617331770550274,
+      "loss": 3.277871322631836,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.7870607376098633,
+      "learning_rate": 0.00016058886788154712,
+      "loss": 3.269728088378906,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.5668118000030518,
+      "learning_rate": 0.00015943188320018388,
+      "loss": 3.210224914550781,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 2.267331123352051,
+      "learning_rate": 0.00015826246505818112,
+      "loss": 3.3089523315429688,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 2.414400815963745,
+      "learning_rate": 0.0001570808581039271,
+      "loss": 3.1313955307006838,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 1.8084098100662231,
+      "learning_rate": 0.00015588730953577335,
+      "loss": 3.372232437133789,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 1.4967864751815796,
+      "learning_rate": 0.0001546820690503195,
+      "loss": 3.3817447662353515,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 1.7170981168746948,
+      "learning_rate": 0.0001534653887901754,
+      "loss": 3.1899662017822266,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 1.7449836730957031,
+      "learning_rate": 0.00015223752329121155,
+      "loss": 3.4185081481933595,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 2.0558719635009766,
+      "learning_rate": 0.00015099872942930887,
+      "loss": 3.4337844848632812,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 1.402202844619751,
+      "learning_rate": 0.0001497492663666189,
+      "loss": 3.000233268737793,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 1.575924038887024,
+      "learning_rate": 0.0001484893954973458,
+      "loss": 3.401926803588867,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.8477239608764648,
+      "learning_rate": 0.00014721938039306134,
+      "loss": 3.143557548522949,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.3538575172424316,
+      "learning_rate": 0.00014593948674756417,
+      "loss": 3.316628265380859,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 1.7879822254180908,
+      "learning_rate": 0.0001446499823212954,
+      "loss": 3.032166862487793,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.9181793928146362,
+      "learning_rate": 0.00014335113688532182,
+      "loss": 3.029685401916504,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 1.4723820686340332,
+      "learning_rate": 0.00014204322216489814,
+      "loss": 3.1593692779541014,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 1.5803518295288086,
+      "learning_rate": 0.00014072651178262096,
+      "loss": 3.1910980224609373,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 1.4509774446487427,
+      "learning_rate": 0.00013940128120118516,
+      "loss": 3.3486377716064455,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 1.2490512132644653,
+      "learning_rate": 0.00013806780766575588,
+      "loss": 3.127117919921875,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 1.3815803527832031,
+      "learning_rate": 0.0001367263701459674,
+      "loss": 3.3135940551757814,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 1.6695923805236816,
+      "learning_rate": 0.00013537724927756094,
+      "loss": 3.24918212890625,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 1.6700210571289062,
+      "learning_rate": 0.00013402072730367475,
+      "loss": 3.1998600006103515,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 1.458272099494934,
+      "learning_rate": 0.0001326570880157967,
+      "loss": 3.2691417694091798,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 1.6035007238388062,
+      "learning_rate": 0.00013128661669439416,
+      "loss": 3.191153717041016,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 1.3588290214538574,
+      "learning_rate": 0.00012990960004923154,
+      "loss": 3.177682113647461,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.3084040880203247,
+      "learning_rate": 0.0001285263261593893,
+      "loss": 3.0778690338134767,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.7576879262924194,
+      "learning_rate": 0.000127137084412996,
+      "loss": 3.063307189941406,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 1.61252760887146,
+      "learning_rate": 0.0001257421654466872,
+      "loss": 3.2951290130615236,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.5347871780395508,
+      "learning_rate": 0.0001243418610848024,
+      "loss": 3.1931894302368162,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 2.2056219577789307,
+      "learning_rate": 0.00012293646427833404,
+      "loss": 3.3431358337402344,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 1.6683447360992432,
+      "learning_rate": 0.00012152626904364067,
+      "loss": 3.057468795776367,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 1.455543875694275,
+      "learning_rate": 0.00012011157040093696,
+      "loss": 3.289105987548828,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 1.7347569465637207,
+      "learning_rate": 0.00011869266431257422,
+      "loss": 2.9250898361206055,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 1.3640249967575073,
+      "learning_rate": 0.00011726984762112328,
+      "loss": 3.325461578369141,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 1.4054839611053467,
+      "learning_rate": 0.00011584341798727365,
+      "loss": 3.168132019042969,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 1.2563236951828003,
+      "learning_rate": 0.0001144136738275612,
+      "loss": 3.237269973754883,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 2.893691062927246,
+      "learning_rate": 0.00011298091425193806,
+      "loss": 3.241581344604492,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 1.9952828884124756,
+      "learning_rate": 0.00011154543900119698,
+      "loss": 3.1403961181640625,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 1.573317050933838,
+      "learning_rate": 0.00011010754838426428,
+      "loss": 3.106332778930664,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 2.1935744285583496,
+      "learning_rate": 0.00010866754321537338,
+      "loss": 3.1031381607055666,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 1.5979467630386353,
+      "learning_rate": 0.00010722572475113315,
+      "loss": 3.1226659774780274,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 1.1873749494552612,
+      "learning_rate": 0.00010578239462750321,
+      "loss": 3.1140846252441405,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 1.8321475982666016,
+      "learning_rate": 0.00010433785479669038,
+      "loss": 3.0106977462768554,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 1.3288156986236572,
+      "learning_rate": 0.00010289240746397832,
+      "loss": 2.8980552673339846,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 1.416706919670105,
+      "learning_rate": 0.00010144635502450508,
+      "loss": 3.141121673583984,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 1.9086672067642212,
+      "learning_rate": 0.0001,
+      "loss": 3.282766342163086,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 1.4448398351669312,
+      "learning_rate": 9.855364497549496e-05,
+      "loss": 2.973546600341797,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 1.2777249813079834,
+      "learning_rate": 9.71075925360217e-05,
+      "loss": 3.145435905456543,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 1.276002287864685,
+      "learning_rate": 9.566214520330966e-05,
+      "loss": 3.2174652099609373,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 1.4613112211227417,
+      "learning_rate": 9.421760537249682e-05,
+      "loss": 3.17722110748291,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 1.1228781938552856,
+      "learning_rate": 9.277427524886689e-05,
+      "loss": 2.98958740234375,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 1.1255987882614136,
+      "learning_rate": 9.133245678462663e-05,
+      "loss": 3.1424510955810545,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.194167137145996,
+      "learning_rate": 8.989245161573576e-05,
+      "loss": 3.0195308685302735,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 1.864384412765503,
+      "learning_rate": 8.845456099880303e-05,
+      "loss": 2.997172737121582,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 1.241004467010498,
+      "learning_rate": 8.701908574806197e-05,
+      "loss": 2.9221866607666014,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 1.1507432460784912,
+      "learning_rate": 8.55863261724388e-05,
+      "loss": 2.9324596405029295,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.287448763847351,
+      "learning_rate": 8.415658201272636e-05,
+      "loss": 2.975849914550781,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 1.6067427396774292,
+      "learning_rate": 8.273015237887673e-05,
+      "loss": 2.980916213989258,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.4008774757385254,
+      "learning_rate": 8.130733568742579e-05,
+      "loss": 2.8944910049438475,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 1.2580070495605469,
+      "learning_rate": 7.988842959906306e-05,
+      "loss": 2.969981384277344,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 1.4962762594223022,
+      "learning_rate": 7.847373095635937e-05,
+      "loss": 3.1294715881347654,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 1.2535191774368286,
+      "learning_rate": 7.706353572166597e-05,
+      "loss": 3.066376876831055,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.8347238898277283,
+      "learning_rate": 7.565813891519765e-05,
+      "loss": 3.0157793045043944,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 1.2819799184799194,
+      "learning_rate": 7.425783455331281e-05,
+      "loss": 3.0389671325683594,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 1.1498677730560303,
+      "learning_rate": 7.2862915587004e-05,
+      "loss": 2.986423301696777,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.1965627670288086,
+      "learning_rate": 7.147367384061074e-05,
+      "loss": 2.941807746887207,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 1.8370661735534668,
+      "learning_rate": 7.009039995076844e-05,
+      "loss": 2.658160400390625,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.418321132659912,
+      "learning_rate": 6.871338330560587e-05,
+      "loss": 3.019854736328125,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 1.3217508792877197,
+      "learning_rate": 6.734291198420333e-05,
+      "loss": 2.8131311416625975,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.207661747932434,
+      "learning_rate": 6.597927269632526e-05,
+      "loss": 3.063412666320801,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 1.1648913621902466,
+      "learning_rate": 6.462275072243908e-05,
+      "loss": 3.0537391662597657,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 1.1196516752243042,
+      "learning_rate": 6.327362985403265e-05,
+      "loss": 3.0568498611450194,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 1.1875085830688477,
+      "learning_rate": 6.193219233424414e-05,
+      "loss": 2.8038040161132813,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.5340442657470703,
+      "learning_rate": 6.059871879881487e-05,
+      "loss": 2.922003746032715,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 1.4385885000228882,
+      "learning_rate": 5.927348821737906e-05,
+      "loss": 2.950858306884766,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 1.0835936069488525,
+      "learning_rate": 5.795677783510187e-05,
+      "loss": 2.898311996459961,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 1.4508082866668701,
+      "learning_rate": 5.664886311467821e-05,
+      "loss": 3.036098670959473,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.0718936920166016,
+      "learning_rate": 5.535001767870463e-05,
+      "loss": 2.8239120483398437,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 1.3246824741363525,
+      "learning_rate": 5.406051325243586e-05,
+      "loss": 2.9098350524902346,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.3559950590133667,
+      "learning_rate": 5.278061960693865e-05,
+      "loss": 2.7230411529541017,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 1.3093736171722412,
+      "learning_rate": 5.151060450265419e-05,
+      "loss": 2.998219871520996,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 1.1871122121810913,
+      "learning_rate": 5.025073363338111e-05,
+      "loss": 2.8021818161010743,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.5887531042099,
+      "learning_rate": 4.900127057069116e-05,
+      "loss": 3.138778305053711,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 1.0622819662094116,
+      "learning_rate": 4.776247670878847e-05,
+      "loss": 2.8708589553833006,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.0821974277496338,
+      "learning_rate": 4.65346112098246e-05,
+      "loss": 2.923636054992676,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 1.2138561010360718,
+      "learning_rate": 4.5317930949680524e-05,
+      "loss": 3.0319366455078125,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 1.3472957611083984,
+      "learning_rate": 4.41126904642267e-05,
+      "loss": 2.7567047119140624,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 1.985767126083374,
+      "learning_rate": 4.291914189607297e-05,
+      "loss": 3.056001663208008,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 1.1290528774261475,
+      "learning_rate": 4.17375349418189e-05,
+      "loss": 3.075904655456543,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 1.2921662330627441,
+      "learning_rate": 4.056811679981609e-05,
+      "loss": 2.8820091247558595,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 1.1284035444259644,
+      "learning_rate": 3.9411132118452896e-05,
+      "loss": 2.897430419921875,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1220782995224,
+      "learning_rate": 3.826682294497265e-05,
+      "loss": 2.782516860961914,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 1.0530436038970947,
+      "learning_rate": 3.7135428674836184e-05,
+      "loss": 2.7634408950805662,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 1.3937631845474243,
+      "learning_rate": 3.6017186001639036e-05,
+      "loss": 2.894866180419922,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 1.00092613697052,
+      "learning_rate": 3.491232886759398e-05,
+      "loss": 2.6879959106445312,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 1.1270594596862793,
+      "learning_rate": 3.382108841458904e-05,
+      "loss": 3.027115821838379,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 1.3118301630020142,
+      "learning_rate": 3.274369293583121e-05,
+      "loss": 2.8866352081298827,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 1.2168943881988525,
+      "learning_rate": 3.1680367828086676e-05,
+      "loss": 2.8968345642089846,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 1.0909639596939087,
+      "learning_rate": 3.063133554452645e-05,
+      "loss": 2.9398611068725584,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 1.1608167886734009,
+      "learning_rate": 2.9596815548187908e-05,
+      "loss": 2.842852783203125,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 1.511507511138916,
+      "learning_rate": 2.8577024266062146e-05,
+      "loss": 2.77429141998291,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 1.1318535804748535,
+      "learning_rate": 2.7572175043816206e-05,
+      "loss": 2.8206886291503905,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.9189174771308899,
+      "learning_rate": 2.6582478101160167e-05,
+      "loss": 2.9603801727294923,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.180462121963501,
+      "learning_rate": 2.560814048786799e-05,
+      "loss": 2.8037836074829103,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 1.1114805936813354,
+      "learning_rate": 2.4649366040461597e-05,
+      "loss": 2.941350746154785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 1.31864595413208,
+      "learning_rate": 2.3706355339567286e-05,
+      "loss": 3.0531469345092774,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.9089719653129578,
+      "learning_rate": 2.2779305667953154e-05,
+      "loss": 2.746686553955078,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 1.0223416090011597,
+      "learning_rate": 2.186841096925657e-05,
+      "loss": 2.8307025909423826,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 1.27829909324646,
+      "learning_rate": 2.097386180741019e-05,
+      "loss": 2.928578758239746,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 1.3367505073547363,
+      "learning_rate": 2.0095845326774697e-05,
+      "loss": 2.8606313705444335,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 1.1823525428771973,
+      "learning_rate": 1.9234545212987688e-05,
+      "loss": 3.041524124145508,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 1.2242523431777954,
+      "learning_rate": 1.8390141654535265e-05,
+      "loss": 3.262685775756836,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.9626563191413879,
+      "learning_rate": 1.756281130505595e-05,
+      "loss": 2.824736976623535,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.9038745760917664,
+      "learning_rate": 1.6752727246383582e-05,
+      "loss": 3.0037527084350586,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 1.0726526975631714,
+      "learning_rate": 1.5960058952337887e-05,
+      "loss": 2.818869781494141,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 1.0472805500030518,
+      "learning_rate": 1.518497225326958e-05,
+      "loss": 2.9350055694580077,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 1.1116654872894287,
+      "learning_rate": 1.4427629301367773e-05,
+      "loss": 2.903761291503906,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.9501079320907593,
+      "learning_rate": 1.3688188536736968e-05,
+      "loss": 2.784720611572266,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 1.109456181526184,
+      "learning_rate": 1.2966804654250465e-05,
+      "loss": 3.1309579849243163,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 1.1933435201644897,
+      "learning_rate": 1.2263628571187414e-05,
+      "loss": 3.0474115371704102,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 1.4391895532608032,
+      "learning_rate": 1.1578807395660207e-05,
+      "loss": 3.0081947326660154,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.9554082155227661,
+      "learning_rate": 1.0912484395838585e-05,
+      "loss": 2.9179988861083985,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 1.1937369108200073,
+      "learning_rate": 1.026479896997723e-05,
+      "loss": 2.705255126953125,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.9862006306648254,
+      "learning_rate": 9.635886617252975e-06,
+      "loss": 2.745578384399414,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 1.1251375675201416,
+      "learning_rate": 9.025878909417552e-06,
+      "loss": 2.965798187255859,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 1.0850991010665894,
+      "learning_rate": 8.434903463272215e-06,
+      "loss": 2.9574241638183594,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 1.476346731185913,
+      "learning_rate": 7.86308391396956e-06,
+      "loss": 2.7245121002197266,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 1.181896686553955,
+      "learning_rate": 7.310539889148404e-06,
+      "loss": 3.0273662567138673,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.4082521200180054,
+      "learning_rate": 6.777386983907152e-06,
+      "loss": 2.9661121368408203,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 1.110843539237976,
+      "learning_rate": 6.263736736620551e-06,
+      "loss": 2.999568557739258,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1253182888031006,
+      "learning_rate": 5.769696605605379e-06,
+      "loss": 2.998799705505371,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.089568853378296,
+      "learning_rate": 5.295369946639628e-06,
+      "loss": 2.6438697814941405,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 1.1276156902313232,
+      "learning_rate": 4.840855991339799e-06,
+      "loss": 2.9343143463134767,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 1.0737465620040894,
+      "learning_rate": 4.406249826401309e-06,
+      "loss": 3.080368995666504,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.8834953904151917,
+      "learning_rate": 3.991642373705695e-06,
+      "loss": 2.917354965209961,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 1.0340644121170044,
+      "learning_rate": 3.5971203712993894e-06,
+      "loss": 2.9122617721557615,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 1.1595566272735596,
+      "learning_rate": 3.2227663552476194e-06,
+      "loss": 2.817066764831543,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 1.0316624641418457,
+      "learning_rate": 2.868658642367439e-06,
+      "loss": 2.764511489868164,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 1.1504026651382446,
+      "learning_rate": 2.5348713138434564e-06,
+      "loss": 2.9506311416625977,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 1.0357000827789307,
+      "learning_rate": 2.2214741997296117e-06,
+      "loss": 2.817026901245117,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8134933710098267,
+      "learning_rate": 1.928532864340438e-06,
+      "loss": 2.7651405334472656,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 1.3359462022781372,
+      "learning_rate": 1.6561085925346332e-06,
+      "loss": 2.8869171142578125,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 1.4075947999954224,
+      "learning_rate": 1.4042583768939298e-06,
+      "loss": 3.0473421096801756,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7921971082687378,
+      "learning_rate": 1.1730349057999412e-06,
+      "loss": 2.8001834869384767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 1.0570473670959473,
+      "learning_rate": 9.624865524115346e-07,
+      "loss": 2.9640541076660156,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 1.0891289710998535,
+      "learning_rate": 7.726573645448487e-07,
+      "loss": 3.0531845092773438,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 1.0408469438552856,
+      "learning_rate": 6.03587055458299e-07,
+      "loss": 2.9867210388183594,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.054602861404419,
+      "learning_rate": 4.5531099554435576e-07,
+      "loss": 2.8087078094482423,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 1.1407268047332764,
+      "learning_rate": 3.278602049298418e-07,
+      "loss": 3.1718997955322266,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.9369794130325317,
+      "learning_rate": 2.2126134698637892e-07,
+      "loss": 2.9225208282470705,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 1.6008250713348389,
+      "learning_rate": 1.3553672275230523e-07,
+      "loss": 3.1131832122802736,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.9389436841011047,
+      "learning_rate": 7.070426626711868e-08,
+      "loss": 2.8224658966064453,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.8056095242500305,
+      "learning_rate": 2.677754081961137e-08,
+      "loss": 2.772154998779297,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.3219596147537231,
+      "learning_rate": 3.7657361103837776e-09,
+      "loss": 3.1340938568115235,
+      "step": 1205
+    },
+    {
+      "epoch": 1.0,
+      "step": 1207,
+      "total_flos": 2.532249769992768e+17,
+      "train_loss": 3.365502968245335,
+      "train_runtime": 20256.5023,
+      "train_samples_per_second": 0.06,
+      "train_steps_per_second": 0.06
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532249769992768e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_lr2e4/training_args.bin b/MOSS_Speaker-RoPE/output_lr2e4/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..a111ea34c5311affa93d9957197d736abc2b0f3b
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_lr2e4/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c303d1b2cafcf35829f18c90c85ffa34cd0d19313509785e9189f8bee8b2e432
+size 5201
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/all_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..9498c854b59b479357d77f3032ea9ae31f86a273
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/all_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.5323923576872166e+17,
+    "train_loss": 0.43766265026083045,
+    "train_runtime": 20328.9815,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.059
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..70fa646f0400625ea034aea5a35a4e8ea8e1772e
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1200/trainer_state.json
@@ -0,0 +1,1714 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9942004971002486,
+  "eval_steps": 500,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 20.629674911499023,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.8916114807128906,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 12.812925338745117,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.558007287979126,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.200536251068115,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.6662204742431641,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 1.363626480102539,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.45300092697143557,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.3842880725860596,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.494948148727417,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.388489842414856,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.4781633377075195,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.5891152024269104,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.48348569869995117,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.9562617540359497,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.4969147205352783,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.6433483958244324,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4836277008056641,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.898301362991333,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4123377799987793,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.8125119805335999,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.45714097023010253,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.5890400409698486,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.4525331497192383,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.6006086468696594,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.3964966297149658,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 1.017815113067627,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37221901416778563,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.9333603978157043,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.4029701232910156,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.0475515127182007,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.45854949951171875,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 1.0967167615890503,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.455504035949707,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.7258841395378113,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7798455238342286,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.0904923677444458,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.46855831146240234,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.924197256565094,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.4880524635314941,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 1.9587433338165283,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.463060998916626,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6440144181251526,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48012351989746094,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8251569867134094,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.4483522415161133,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.8789721727371216,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4580700397491455,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8313543796539307,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.46596560478210447,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.6774608492851257,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4806799411773682,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.6783014535903931,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4560887336730957,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.7160524129867554,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.4927361011505127,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5270054340362549,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.42853679656982424,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 1.0818812847137451,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5378652572631836,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.968137800693512,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.48653225898742675,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.8479903936386108,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.5200669765472412,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7051109075546265,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.4588489532470703,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 2.205345630645752,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.5904168128967285,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.8751126527786255,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.43442707061767577,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.671466052532196,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3809401512145996,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 1.235303282737732,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4351661205291748,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.5449215769767761,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.41109681129455566,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6575606465339661,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.46851186752319335,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.528086245059967,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40171051025390625,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.44805943965911865,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.3575094223022461,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.6913977861404419,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.4208680152893066,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6070751547813416,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4100965976715088,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.7453505396842957,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5092792510986328,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.718826949596405,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5120072841644288,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.8064695596694946,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.43071684837341306,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.6657348275184631,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.43650312423706056,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6227285861968994,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.4965176582336426,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6354798078536987,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.4135477542877197,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.8802199959754944,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.4657131671905518,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6120206713676453,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5105055332183838,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.6608315706253052,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48976764678955076,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6136434078216553,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39600679874420164,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7732479572296143,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.43799166679382323,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8353448510169983,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.39786520004272463,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5929117798805237,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4511585235595703,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7609494924545288,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.460142993927002,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.4482679069042206,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.42333049774169923,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5880385637283325,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39257287979125977,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4800879955291748,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46072850227355955,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5711155533790588,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4340324401855469,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.6473472118377686,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4806147575378418,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.9981988668441772,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.45314898490905764,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.9140734076499939,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48151440620422364,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.7532357573509216,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4129056453704834,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.9811023473739624,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.484523344039917,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.4147282838821411,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.5427152633666992,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 1.0163606405258179,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.4134209156036377,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.6049103140830994,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.40175676345825195,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.7045316696166992,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40282230377197265,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.6261049509048462,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3981405973434448,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.287496566772461,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4517494201660156,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6869074702262878,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.44563779830932615,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6493797302246094,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4556128978729248,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5839359760284424,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.43351993560791013,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3983668088912964,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.4440763473510742,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6596688032150269,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5530457973480225,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6001258492469788,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4752801418304443,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.405813604593277,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.38340303897857664,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.8015041351318359,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.4637146949768066,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.6381627321243286,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42714495658874513,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5829190015792847,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.3956311702728271,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8698112964630127,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4846199989318848,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6768237352371216,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.47880539894104,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.5710030198097229,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4479952335357666,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6229125261306763,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.433013916015625,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6174870133399963,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4820831298828125,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.9886339902877808,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4312156677246094,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7721105217933655,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4745286464691162,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.8215830326080322,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.43494420051574706,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8584961295127869,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46215004920959474,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.483641117811203,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.37451136112213135,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.567133903503418,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.4840177059173584,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.6834204196929932,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.46247282028198244,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.8395087122917175,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.36043124198913573,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3639385998249054,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42089052200317384,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4404556453227997,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3542492866516113,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6046218276023865,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.38021628856658934,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.6285887956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6484775066375732,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9363073110580444,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.41137142181396485,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4143483340740204,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3924715518951416,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.2242801189422607,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.4372579574584961,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.49445945024490356,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.4041281700134277,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5578067898750305,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.44759469032287597,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6338609457015991,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.3942832946777344,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6305315494537354,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.3744609832763672,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6225669980049133,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4644033432006836,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4533715844154358,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43013477325439453,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.668293297290802,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5000359058380127,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6384305357933044,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4234946250915527,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.35919633507728577,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4122346878051758,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.5707134008407593,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4068869113922119,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.7894991636276245,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4764272689819336,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.7096588611602783,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.40894289016723634,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.6279922723770142,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4551478385925293,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9392326474189758,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42044363021850584,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.8099265098571777,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.421197509765625,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.407980740070343,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45879392623901366,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.6435403823852539,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7919346332550049,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.5578521490097046,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4143537044525146,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5052011013031006,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.38072867393493653,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.6519150733947754,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4297952175140381,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.9843361377716064,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3838667869567871,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.7498537302017212,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.40416569709777833,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.6527976393699646,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.4134161949157715,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.8321830630302429,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.43797922134399414,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.6492611169815063,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.4532886028289795,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.6094037294387817,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43567543029785155,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.551053524017334,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4400018692016602,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6376934051513672,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42303104400634767,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.9007910490036011,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45666775703430174,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.650941014289856,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4667072296142578,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.48627233505249023,
+      "learning_rate": 5e-05,
+      "loss": 0.3643670558929443,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7920517325401306,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44395933151245115,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.5550063252449036,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.36930572986602783,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.6230601668357849,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.4727597236633301,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.6426054835319519,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.460523796081543,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5865359902381897,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.4252904415130615,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.5453794598579407,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4039918422698975,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.0684459209442139,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42879571914672854,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.8819708824157715,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35405232906341555,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5523223280906677,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36108903884887694,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.49861323833465576,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4754959583282471,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.2883014678955078,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.38763861656188964,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5406184792518616,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.39906909465789797,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8991988301277161,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.418521785736084,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5573598742485046,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43770723342895507,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.7751765847206116,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.36180825233459474,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5557352304458618,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.4191638946533203,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 1.012902021408081,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43454732894897463,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.8307779431343079,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.3497183799743652,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.6312240958213806,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.43721756935119627,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.9830090403556824,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48459711074829104,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.4116923213005066,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31733293533325196,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.0353782176971436,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.43854188919067383,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.7071158289909363,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.4189001560211182,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.6019755601882935,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7713140487670899,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5350252389907837,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3926787137985229,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.37985295057296753,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.4014743328094482,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.6357853412628174,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4581602096557617,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.5001391768455505,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.3605159044265747,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9979525804519653,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5704981803894043,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.6091923713684082,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3738471269607544,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4976308345794678,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43538880348205566,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.9942322969436646,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554206609725952,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6866982579231262,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.4553103446960449,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.89406818151474,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.43181838989257815,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.5093903541564941,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4385825157165527,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5549860596656799,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.329516339302063,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.4509713649749756,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3463990926742554,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5200580358505249,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.40448341369628904,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.1131764650344849,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.426401424407959,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7889172434806824,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38459455966949463,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.517167866230011,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.36908698081970215,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5234260559082031,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3863369941711426,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.7236451506614685,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40941739082336426,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4438631236553192,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.34200191497802734,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.39247673749923706,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4146144390106201,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1701165437698364,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.4748171329498291,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5402036309242249,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.3901249170303345,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6763427257537842,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.4359886646270752,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5417786240577698,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39662876129150393,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.7444621920585632,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3708503723144531,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.7745838165283203,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3678457736968994,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.4944072961807251,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3787250280380249,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.7562234997749329,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.4543753623962402,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5324202179908752,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.34689984321594236,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6181265711784363,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.396760082244873,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.5689008831977844,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.3692843198776245,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.631606936454773,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.683953332901001,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.7201916575431824,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.37404820919036863,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5721992254257202,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7489796638488769,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.7852861881256104,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024354457855225,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.5761847496032715,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4159713268280029,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.4608430564403534,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.38759047985076905,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.4075900614261627,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.38979687690734866,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6070588231086731,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.43787498474121095,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.5205054879188538,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39637553691864014,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.8867539763450623,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4684460639953613,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.7465023398399353,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3949731826782227,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.6426427960395813,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4073126792907715,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.6218783259391785,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.43186631202697756,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6543543934822083,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.35069923400878905,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8917542099952698,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47125835418701173,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4148687422275543,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.38721065521240233,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6250177621841431,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44888858795166015,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.4487530291080475,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.37155201435089114,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.450059711933136,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44730415344238283,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.858016312122345,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4445347309112549,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.5695084929466248,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.4245354175567627,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4118499159812927,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.39943842887878417,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.7165275812149048,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.43923196792602537,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.5231788754463196,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.41583733558654784,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4489861726760864,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4420301914215088,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.7916709184646606,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.415742826461792,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.1566158533096313,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4660985469818115,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5106304287910461,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.36864919662475587,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.6080639362335205,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.44466662406921387,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.5853272080421448,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.3511647701263428,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.41179347038269043,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.4063854217529297,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5927942991256714,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42436866760253905,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7885688543319702,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3807806968688965,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5567767024040222,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.4070634365081787,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.7330124378204346,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41397967338562014,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.45662936568260193,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3888988494873047,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.48038941621780396,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3652131795883179,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6213480830192566,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.3998314142227173,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.7687107920646667,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.45170745849609373,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6509118676185608,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42131795883178713,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6040301322937012,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3819763422012329,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.8355879783630371,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42635231018066405,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.548719048500061,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.35939807891845704,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5332063436508179,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.4174611568450928,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5104030966758728,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.35938918590545654,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.0479686260223389,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.353275465965271,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6005430817604065,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.44223575592041015,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6831442713737488,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36453771591186523,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.6081545948982239,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.35844419002532957,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.4227214455604553,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.399729585647583,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.9676080942153931,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.43811402320861814,
+      "step": 1200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.518258444938741e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..24a88fbc20f89eb524da8b0ddd1b4d786fe96308
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/checkpoint-1207/trainer_state.json
@@ -0,0 +1,1721 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 20.629674911499023,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.8916114807128906,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 12.812925338745117,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.558007287979126,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.200536251068115,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.6662204742431641,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 1.363626480102539,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.45300092697143557,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.3842880725860596,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.494948148727417,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.388489842414856,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.4781633377075195,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.5891152024269104,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.48348569869995117,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.9562617540359497,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.4969147205352783,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.6433483958244324,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4836277008056641,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.898301362991333,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4123377799987793,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.8125119805335999,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.45714097023010253,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.5890400409698486,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.4525331497192383,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.6006086468696594,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.3964966297149658,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 1.017815113067627,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37221901416778563,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.9333603978157043,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.4029701232910156,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.0475515127182007,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.45854949951171875,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 1.0967167615890503,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.455504035949707,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.7258841395378113,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7798455238342286,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.0904923677444458,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.46855831146240234,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.924197256565094,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.4880524635314941,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 1.9587433338165283,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.463060998916626,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6440144181251526,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48012351989746094,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8251569867134094,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.4483522415161133,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.8789721727371216,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4580700397491455,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8313543796539307,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.46596560478210447,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.6774608492851257,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4806799411773682,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.6783014535903931,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4560887336730957,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.7160524129867554,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.4927361011505127,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5270054340362549,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.42853679656982424,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 1.0818812847137451,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5378652572631836,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.968137800693512,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.48653225898742675,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.8479903936386108,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.5200669765472412,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7051109075546265,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.4588489532470703,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 2.205345630645752,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.5904168128967285,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.8751126527786255,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.43442707061767577,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.671466052532196,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3809401512145996,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 1.235303282737732,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4351661205291748,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.5449215769767761,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.41109681129455566,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6575606465339661,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.46851186752319335,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.528086245059967,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40171051025390625,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.44805943965911865,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.3575094223022461,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.6913977861404419,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.4208680152893066,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6070751547813416,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4100965976715088,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.7453505396842957,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5092792510986328,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.718826949596405,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5120072841644288,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.8064695596694946,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.43071684837341306,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.6657348275184631,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.43650312423706056,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6227285861968994,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.4965176582336426,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6354798078536987,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.4135477542877197,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.8802199959754944,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.4657131671905518,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6120206713676453,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5105055332183838,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.6608315706253052,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48976764678955076,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6136434078216553,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39600679874420164,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7732479572296143,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.43799166679382323,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8353448510169983,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.39786520004272463,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5929117798805237,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4511585235595703,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7609494924545288,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.460142993927002,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.4482679069042206,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.42333049774169923,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5880385637283325,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39257287979125977,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4800879955291748,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46072850227355955,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5711155533790588,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4340324401855469,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.6473472118377686,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4806147575378418,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.9981988668441772,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.45314898490905764,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.9140734076499939,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48151440620422364,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.7532357573509216,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4129056453704834,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.9811023473739624,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.484523344039917,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.4147282838821411,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.5427152633666992,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 1.0163606405258179,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.4134209156036377,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.6049103140830994,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.40175676345825195,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.7045316696166992,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40282230377197265,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.6261049509048462,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3981405973434448,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.287496566772461,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4517494201660156,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6869074702262878,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.44563779830932615,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6493797302246094,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4556128978729248,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5839359760284424,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.43351993560791013,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3983668088912964,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.4440763473510742,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6596688032150269,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5530457973480225,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6001258492469788,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4752801418304443,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.405813604593277,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.38340303897857664,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.8015041351318359,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.4637146949768066,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.6381627321243286,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42714495658874513,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5829190015792847,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.3956311702728271,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8698112964630127,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4846199989318848,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6768237352371216,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.47880539894104,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.5710030198097229,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4479952335357666,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6229125261306763,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.433013916015625,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6174870133399963,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4820831298828125,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.9886339902877808,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4312156677246094,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7721105217933655,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4745286464691162,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.8215830326080322,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.43494420051574706,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8584961295127869,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46215004920959474,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.483641117811203,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.37451136112213135,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.567133903503418,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.4840177059173584,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.6834204196929932,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.46247282028198244,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.8395087122917175,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.36043124198913573,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3639385998249054,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42089052200317384,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4404556453227997,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3542492866516113,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6046218276023865,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.38021628856658934,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.6285887956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6484775066375732,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9363073110580444,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.41137142181396485,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4143483340740204,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3924715518951416,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.2242801189422607,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.4372579574584961,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.49445945024490356,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.4041281700134277,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5578067898750305,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.44759469032287597,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6338609457015991,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.3942832946777344,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6305315494537354,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.3744609832763672,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6225669980049133,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4644033432006836,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4533715844154358,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43013477325439453,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.668293297290802,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5000359058380127,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6384305357933044,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4234946250915527,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.35919633507728577,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4122346878051758,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.5707134008407593,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4068869113922119,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.7894991636276245,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4764272689819336,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.7096588611602783,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.40894289016723634,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.6279922723770142,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4551478385925293,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9392326474189758,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42044363021850584,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.8099265098571777,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.421197509765625,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.407980740070343,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45879392623901366,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.6435403823852539,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7919346332550049,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.5578521490097046,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4143537044525146,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5052011013031006,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.38072867393493653,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.6519150733947754,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4297952175140381,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.9843361377716064,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3838667869567871,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.7498537302017212,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.40416569709777833,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.6527976393699646,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.4134161949157715,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.8321830630302429,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.43797922134399414,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.6492611169815063,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.4532886028289795,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.6094037294387817,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43567543029785155,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.551053524017334,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4400018692016602,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6376934051513672,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42303104400634767,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.9007910490036011,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45666775703430174,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.650941014289856,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4667072296142578,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.48627233505249023,
+      "learning_rate": 5e-05,
+      "loss": 0.3643670558929443,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7920517325401306,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44395933151245115,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.5550063252449036,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.36930572986602783,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.6230601668357849,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.4727597236633301,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.6426054835319519,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.460523796081543,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5865359902381897,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.4252904415130615,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.5453794598579407,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4039918422698975,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.0684459209442139,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42879571914672854,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.8819708824157715,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35405232906341555,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5523223280906677,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36108903884887694,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.49861323833465576,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4754959583282471,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.2883014678955078,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.38763861656188964,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5406184792518616,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.39906909465789797,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8991988301277161,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.418521785736084,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5573598742485046,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43770723342895507,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.7751765847206116,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.36180825233459474,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5557352304458618,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.4191638946533203,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 1.012902021408081,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43454732894897463,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.8307779431343079,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.3497183799743652,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.6312240958213806,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.43721756935119627,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.9830090403556824,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48459711074829104,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.4116923213005066,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31733293533325196,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.0353782176971436,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.43854188919067383,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.7071158289909363,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.4189001560211182,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.6019755601882935,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7713140487670899,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5350252389907837,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3926787137985229,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.37985295057296753,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.4014743328094482,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.6357853412628174,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4581602096557617,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.5001391768455505,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.3605159044265747,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9979525804519653,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5704981803894043,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.6091923713684082,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3738471269607544,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4976308345794678,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43538880348205566,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.9942322969436646,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554206609725952,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6866982579231262,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.4553103446960449,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.89406818151474,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.43181838989257815,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.5093903541564941,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4385825157165527,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5549860596656799,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.329516339302063,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.4509713649749756,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3463990926742554,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5200580358505249,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.40448341369628904,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.1131764650344849,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.426401424407959,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7889172434806824,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38459455966949463,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.517167866230011,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.36908698081970215,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5234260559082031,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3863369941711426,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.7236451506614685,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40941739082336426,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4438631236553192,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.34200191497802734,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.39247673749923706,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4146144390106201,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1701165437698364,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.4748171329498291,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5402036309242249,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.3901249170303345,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6763427257537842,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.4359886646270752,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5417786240577698,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39662876129150393,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.7444621920585632,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3708503723144531,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.7745838165283203,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3678457736968994,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.4944072961807251,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3787250280380249,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.7562234997749329,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.4543753623962402,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5324202179908752,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.34689984321594236,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6181265711784363,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.396760082244873,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.5689008831977844,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.3692843198776245,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.631606936454773,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.683953332901001,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.7201916575431824,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.37404820919036863,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5721992254257202,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7489796638488769,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.7852861881256104,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024354457855225,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.5761847496032715,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4159713268280029,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.4608430564403534,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.38759047985076905,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.4075900614261627,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.38979687690734866,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6070588231086731,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.43787498474121095,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.5205054879188538,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39637553691864014,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.8867539763450623,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4684460639953613,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.7465023398399353,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3949731826782227,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.6426427960395813,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4073126792907715,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.6218783259391785,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.43186631202697756,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6543543934822083,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.35069923400878905,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8917542099952698,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47125835418701173,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4148687422275543,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.38721065521240233,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6250177621841431,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44888858795166015,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.4487530291080475,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.37155201435089114,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.450059711933136,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44730415344238283,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.858016312122345,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4445347309112549,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.5695084929466248,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.4245354175567627,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4118499159812927,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.39943842887878417,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.7165275812149048,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.43923196792602537,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.5231788754463196,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.41583733558654784,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4489861726760864,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4420301914215088,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.7916709184646606,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.415742826461792,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.1566158533096313,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4660985469818115,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5106304287910461,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.36864919662475587,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.6080639362335205,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.44466662406921387,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.5853272080421448,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.3511647701263428,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.41179347038269043,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.4063854217529297,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5927942991256714,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42436866760253905,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7885688543319702,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3807806968688965,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5567767024040222,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.4070634365081787,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.7330124378204346,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41397967338562014,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.45662936568260193,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3888988494873047,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.48038941621780396,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3652131795883179,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6213480830192566,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.3998314142227173,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.7687107920646667,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.45170745849609373,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6509118676185608,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42131795883178713,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6040301322937012,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3819763422012329,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.8355879783630371,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42635231018066405,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.548719048500061,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.35939807891845704,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5332063436508179,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.4174611568450928,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5104030966758728,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.35938918590545654,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.0479686260223389,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.353275465965271,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6005430817604065,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.44223575592041015,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6831442713737488,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36453771591186523,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.6081545948982239,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.35844419002532957,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.4227214455604553,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.399729585647583,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.9676080942153931,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.43811402320861814,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 0.9380176067352295,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.36945352554321287,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5323923576872166e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/model.safetensors b/MOSS_Speaker-RoPE/output_spk_odd8_s3/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3f2b1b62520ed1500674dfd79f4017b72fd3ec63
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe6a5534156882e11783efeedfe33fb4b18b4b17b6b6ebdc5ab64c2f19c8fc4a
+size 1868024544
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f3fe62e4fcd17442942325dec95aa58e731e4a7
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb21895752d2380b32ae2e2c40905c7aa6c8e55c5e6ba33cd065bf77d4d53f74
+size 11423344
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/train_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..9498c854b59b479357d77f3032ea9ae31f86a273
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.5323923576872166e+17,
+    "train_loss": 0.43766265026083045,
+    "train_runtime": 20328.9815,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.059
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..072557f23999533c4a5d8790eb10ec9c78c2ed69
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/trainer_state.json
@@ -0,0 +1,1730 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 20.629674911499023,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.8916114807128906,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 12.812925338745117,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.558007287979126,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.200536251068115,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.6662204742431641,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 1.363626480102539,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.45300092697143557,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.3842880725860596,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.494948148727417,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.388489842414856,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.4781633377075195,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.5891152024269104,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.48348569869995117,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.9562617540359497,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.4969147205352783,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.6433483958244324,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4836277008056641,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.898301362991333,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4123377799987793,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.8125119805335999,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.45714097023010253,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.5890400409698486,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.4525331497192383,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.6006086468696594,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.3964966297149658,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 1.017815113067627,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37221901416778563,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.9333603978157043,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.4029701232910156,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.0475515127182007,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.45854949951171875,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 1.0967167615890503,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.455504035949707,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.7258841395378113,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7798455238342286,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.0904923677444458,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.46855831146240234,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.924197256565094,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.4880524635314941,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 1.9587433338165283,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.463060998916626,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6440144181251526,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48012351989746094,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8251569867134094,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.4483522415161133,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.8789721727371216,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4580700397491455,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8313543796539307,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.46596560478210447,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.6774608492851257,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4806799411773682,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.6783014535903931,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4560887336730957,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.7160524129867554,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.4927361011505127,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5270054340362549,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.42853679656982424,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 1.0818812847137451,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5378652572631836,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.968137800693512,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.48653225898742675,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.8479903936386108,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.5200669765472412,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7051109075546265,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.4588489532470703,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 2.205345630645752,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.5904168128967285,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.8751126527786255,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.43442707061767577,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.671466052532196,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3809401512145996,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 1.235303282737732,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4351661205291748,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.5449215769767761,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.41109681129455566,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6575606465339661,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.46851186752319335,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.528086245059967,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40171051025390625,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.44805943965911865,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.3575094223022461,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.6913977861404419,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.4208680152893066,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6070751547813416,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4100965976715088,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.7453505396842957,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5092792510986328,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.718826949596405,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5120072841644288,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.8064695596694946,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.43071684837341306,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.6657348275184631,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.43650312423706056,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6227285861968994,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.4965176582336426,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6354798078536987,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.4135477542877197,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.8802199959754944,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.4657131671905518,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6120206713676453,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5105055332183838,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.6608315706253052,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48976764678955076,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6136434078216553,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39600679874420164,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7732479572296143,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.43799166679382323,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8353448510169983,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.39786520004272463,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5929117798805237,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4511585235595703,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7609494924545288,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.460142993927002,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.4482679069042206,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.42333049774169923,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5880385637283325,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39257287979125977,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4800879955291748,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46072850227355955,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5711155533790588,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4340324401855469,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 1.6473472118377686,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4806147575378418,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.9981988668441772,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.45314898490905764,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.9140734076499939,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48151440620422364,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.7532357573509216,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4129056453704834,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.9811023473739624,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.484523344039917,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.4147282838821411,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.5427152633666992,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 1.0163606405258179,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.4134209156036377,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.6049103140830994,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.40175676345825195,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.7045316696166992,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40282230377197265,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.6261049509048462,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3981405973434448,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.287496566772461,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4517494201660156,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6869074702262878,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.44563779830932615,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6493797302246094,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4556128978729248,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5839359760284424,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.43351993560791013,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3983668088912964,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.4440763473510742,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6596688032150269,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5530457973480225,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6001258492469788,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4752801418304443,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.405813604593277,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.38340303897857664,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.8015041351318359,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.4637146949768066,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.6381627321243286,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42714495658874513,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5829190015792847,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.3956311702728271,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8698112964630127,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4846199989318848,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6768237352371216,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.47880539894104,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.5710030198097229,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4479952335357666,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6229125261306763,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.433013916015625,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6174870133399963,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4820831298828125,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 1.9886339902877808,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4312156677246094,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7721105217933655,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4745286464691162,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.8215830326080322,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.43494420051574706,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8584961295127869,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46215004920959474,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.483641117811203,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.37451136112213135,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.567133903503418,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.4840177059173584,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.6834204196929932,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.46247282028198244,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.8395087122917175,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.36043124198913573,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3639385998249054,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42089052200317384,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4404556453227997,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3542492866516113,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6046218276023865,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.38021628856658934,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.6285887956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6484775066375732,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9363073110580444,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.41137142181396485,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4143483340740204,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3924715518951416,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.2242801189422607,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.4372579574584961,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.49445945024490356,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.4041281700134277,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5578067898750305,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.44759469032287597,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6338609457015991,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.3942832946777344,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6305315494537354,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.3744609832763672,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6225669980049133,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4644033432006836,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4533715844154358,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43013477325439453,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.668293297290802,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5000359058380127,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6384305357933044,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4234946250915527,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.35919633507728577,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4122346878051758,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.5707134008407593,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4068869113922119,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 1.7894991636276245,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4764272689819336,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.7096588611602783,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.40894289016723634,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.6279922723770142,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4551478385925293,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9392326474189758,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42044363021850584,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.8099265098571777,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.421197509765625,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.407980740070343,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45879392623901366,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.6435403823852539,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7919346332550049,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.5578521490097046,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4143537044525146,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5052011013031006,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.38072867393493653,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.6519150733947754,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4297952175140381,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.9843361377716064,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3838667869567871,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.7498537302017212,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.40416569709777833,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.6527976393699646,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.4134161949157715,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.8321830630302429,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.43797922134399414,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.6492611169815063,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.4532886028289795,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.6094037294387817,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43567543029785155,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.551053524017334,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4400018692016602,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6376934051513672,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42303104400634767,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.9007910490036011,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45666775703430174,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.650941014289856,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4667072296142578,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.48627233505249023,
+      "learning_rate": 5e-05,
+      "loss": 0.3643670558929443,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7920517325401306,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44395933151245115,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.5550063252449036,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.36930572986602783,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.6230601668357849,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.4727597236633301,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.6426054835319519,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.460523796081543,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5865359902381897,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.4252904415130615,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.5453794598579407,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4039918422698975,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 1.0684459209442139,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42879571914672854,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.8819708824157715,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35405232906341555,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5523223280906677,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36108903884887694,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.49861323833465576,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4754959583282471,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.2883014678955078,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.38763861656188964,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5406184792518616,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.39906909465789797,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8991988301277161,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.418521785736084,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5573598742485046,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43770723342895507,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.7751765847206116,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.36180825233459474,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5557352304458618,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.4191638946533203,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 1.012902021408081,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43454732894897463,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.8307779431343079,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.3497183799743652,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.6312240958213806,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.43721756935119627,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.9830090403556824,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48459711074829104,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.4116923213005066,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31733293533325196,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 1.0353782176971436,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.43854188919067383,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.7071158289909363,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.4189001560211182,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 1.6019755601882935,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7713140487670899,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5350252389907837,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3926787137985229,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.37985295057296753,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.4014743328094482,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.6357853412628174,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4581602096557617,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.5001391768455505,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.3605159044265747,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9979525804519653,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5704981803894043,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.6091923713684082,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3738471269607544,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4976308345794678,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43538880348205566,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.9942322969436646,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554206609725952,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6866982579231262,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.4553103446960449,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.89406818151474,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.43181838989257815,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.5093903541564941,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4385825157165527,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5549860596656799,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.329516339302063,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 1.4509713649749756,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3463990926742554,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5200580358505249,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.40448341369628904,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 1.1131764650344849,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.426401424407959,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7889172434806824,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38459455966949463,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.517167866230011,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.36908698081970215,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5234260559082031,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3863369941711426,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.7236451506614685,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40941739082336426,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4438631236553192,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.34200191497802734,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.39247673749923706,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4146144390106201,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.1701165437698364,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.4748171329498291,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5402036309242249,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.3901249170303345,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6763427257537842,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.4359886646270752,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5417786240577698,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39662876129150393,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.7444621920585632,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3708503723144531,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.7745838165283203,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3678457736968994,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.4944072961807251,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3787250280380249,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.7562234997749329,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.4543753623962402,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5324202179908752,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.34689984321594236,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6181265711784363,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.396760082244873,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.5689008831977844,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.3692843198776245,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.631606936454773,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.683953332901001,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.7201916575431824,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.37404820919036863,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5721992254257202,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7489796638488769,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.7852861881256104,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024354457855225,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.5761847496032715,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4159713268280029,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.4608430564403534,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.38759047985076905,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.4075900614261627,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.38979687690734866,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6070588231086731,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.43787498474121095,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.5205054879188538,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39637553691864014,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.8867539763450623,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4684460639953613,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.7465023398399353,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3949731826782227,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.6426427960395813,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4073126792907715,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.6218783259391785,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.43186631202697756,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6543543934822083,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.35069923400878905,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8917542099952698,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47125835418701173,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4148687422275543,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.38721065521240233,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6250177621841431,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44888858795166015,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.4487530291080475,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.37155201435089114,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.450059711933136,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44730415344238283,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.858016312122345,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4445347309112549,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.5695084929466248,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.4245354175567627,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4118499159812927,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.39943842887878417,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.7165275812149048,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.43923196792602537,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.5231788754463196,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.41583733558654784,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4489861726760864,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4420301914215088,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.7916709184646606,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.415742826461792,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 1.1566158533096313,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4660985469818115,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5106304287910461,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.36864919662475587,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.6080639362335205,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.44466662406921387,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.5853272080421448,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.3511647701263428,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.41179347038269043,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.4063854217529297,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5927942991256714,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42436866760253905,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7885688543319702,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3807806968688965,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5567767024040222,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.4070634365081787,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.7330124378204346,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41397967338562014,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.45662936568260193,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3888988494873047,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.48038941621780396,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3652131795883179,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6213480830192566,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.3998314142227173,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.7687107920646667,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.45170745849609373,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6509118676185608,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42131795883178713,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6040301322937012,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3819763422012329,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.8355879783630371,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42635231018066405,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.548719048500061,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.35939807891845704,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5332063436508179,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.4174611568450928,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5104030966758728,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.35938918590545654,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 1.0479686260223389,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.353275465965271,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6005430817604065,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.44223575592041015,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6831442713737488,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36453771591186523,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.6081545948982239,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.35844419002532957,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.4227214455604553,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.399729585647583,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.9676080942153931,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.43811402320861814,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 0.9380176067352295,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.36945352554321287,
+      "step": 1205
+    },
+    {
+      "epoch": 1.0,
+      "step": 1207,
+      "total_flos": 2.5323923576872166e+17,
+      "train_loss": 0.43766265026083045,
+      "train_runtime": 20328.9815,
+      "train_samples_per_second": 0.059,
+      "train_steps_per_second": 0.059
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5323923576872166e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3/training_args.bin b/MOSS_Speaker-RoPE/output_spk_odd8_s3/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..403998e3e7e093640339afb11ead65d5c78ef830
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6c93d4e24f3dbcb6d51b3baf817d83d0ead3706ab19f7f34fac257d122e9e7
+size 5201
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..59e8c42d7f8f5f31e6f2e52557a63a1761b68ba2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-400/trainer_state.json
@@ -0,0 +1,594 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.33140016570008285,
+  "eval_steps": 500,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 31.104061126708984,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 1.1910821914672851,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 35.740692138671875,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.7337178230285645,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 9.301889419555664,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.7512782096862793,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 2.6755707263946533,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.4849919319152832,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 1.123673677444458,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.529819917678833,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 2.8139865398406982,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.5005036354064941,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 1.766931414604187,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.4980137825012207,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 1.1081942319869995,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.5065455913543702,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 1.07539963722229,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4957615852355957,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 1.0425145626068115,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.42017226219177245,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.8854573965072632,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.46346406936645507,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.678348183631897,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.4590586185455322,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.6650195121765137,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.3999401807785034,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.9779027700424194,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37582452297210694,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 1.0380560159683228,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.4077299118041992,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.2877566814422607,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.46419506072998046,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 1.029144287109375,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.4615780830383301,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.5973961353302002,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7924308776855469,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.3301033973693848,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.47766985893249514,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 1.078078269958496,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.500474214553833,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 1.0514358282089233,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.47164087295532225,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6962097883224487,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.4897346019744873,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.9387894868850708,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.45862503051757814,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 1.0993168354034424,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.463796329498291,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8082759380340576,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.47555227279663087,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.7035478353500366,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.49724931716918946,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.6950880885124207,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4550027370452881,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 1.0088214874267578,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.5083928108215332,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.747430145740509,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.43213858604431155,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.9024979472160339,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5470420360565186,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 1.1015739440917969,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.4933629989624023,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.8720569610595703,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.5295196533203125,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7969361543655396,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.4577418327331543,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.101267099380493,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.6081419467926026,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.7658870816230774,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.4367526531219482,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.6250601410865784,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3846379041671753,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.8455803990364075,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4439352035522461,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.5648259520530701,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.41419034004211425,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 1.176491618156433,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.4723679542541504,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.5706663727760315,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.4028864860534668,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.5537962317466736,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.3607978820800781,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.6196736097335815,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.42556142807006836,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6695680022239685,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.41333789825439454,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.655034065246582,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5096014976501465,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.5938177704811096,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5216481685638428,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.7804147601127625,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.43359966278076173,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.7048928141593933,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.4395446300506592,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6047908067703247,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.4992673397064209,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6019694209098816,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.41515302658081055,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.8708873391151428,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.4704007625579834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6083199381828308,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5111989974975586,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5877307653427124,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48946352005004884,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6169144511222839,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39699459075927734,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7712729573249817,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.43967700004577637,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.7823882102966309,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.39926810264587403,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6713236570358276,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.454238748550415,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.8165543079376221,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.46351184844970705,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.37962615489959717,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.4180287837982178,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.6618972420692444,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.3922520637512207,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.42513608932495117,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46528077125549316,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5760522484779358,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4353321075439453,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.7091104984283447,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4817354679107666,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.7984012365341187,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.45444478988647463,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7693517804145813,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.4780622959136963,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5142762660980225,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4113470554351807,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.682898223400116,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.48437914848327634,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.3431150913238525,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.5317914962768555,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.7310293316841125,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.41095829010009766,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.564328134059906,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.4005306243896484,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6681087017059326,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.399173641204834,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.5092662572860718,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3901823043823242,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.0772058963775635,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4496616840362549,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.596607506275177,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.43782577514648435,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6527637243270874,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4539346218109131,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.6109674572944641,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.4309091091156006,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.4367820918560028,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.4418576717376709,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6753432154655457,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5508285522460937,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.605097234249115,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.47382965087890627,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.4321838617324829,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.3830132961273193,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6553069353103638,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.4628459930419922,
+      "step": 400
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.375348468992243e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..35b1ef73092d407f4ae5b95eab3113bba007fa39
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_old/checkpoint-600/trainer_state.json
@@ -0,0 +1,874 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4971002485501243,
+  "eval_steps": 500,
+  "global_step": 600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 31.104061126708984,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 1.1910821914672851,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 35.740692138671875,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.7337178230285645,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 9.301889419555664,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.7512782096862793,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 2.6755707263946533,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.4849919319152832,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 1.123673677444458,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.529819917678833,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 2.8139865398406982,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.5005036354064941,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 1.766931414604187,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.4980137825012207,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 1.1081942319869995,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.5065455913543702,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 1.07539963722229,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4957615852355957,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 1.0425145626068115,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.42017226219177245,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.8854573965072632,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.46346406936645507,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.678348183631897,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.4590586185455322,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.6650195121765137,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.3999401807785034,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.9779027700424194,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37582452297210694,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 1.0380560159683228,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.4077299118041992,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.2877566814422607,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.46419506072998046,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 1.029144287109375,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.4615780830383301,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.5973961353302002,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7924308776855469,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.3301033973693848,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.47766985893249514,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 1.078078269958496,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.500474214553833,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 1.0514358282089233,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.47164087295532225,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6962097883224487,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.4897346019744873,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.9387894868850708,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.45862503051757814,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 1.0993168354034424,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.463796329498291,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8082759380340576,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.47555227279663087,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.7035478353500366,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.49724931716918946,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.6950880885124207,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4550027370452881,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 1.0088214874267578,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.5083928108215332,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.747430145740509,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.43213858604431155,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.9024979472160339,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5470420360565186,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 1.1015739440917969,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.4933629989624023,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.8720569610595703,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.5295196533203125,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7969361543655396,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.4577418327331543,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.101267099380493,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.6081419467926026,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.7658870816230774,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.4367526531219482,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.6250601410865784,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3846379041671753,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.8455803990364075,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4439352035522461,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.5648259520530701,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.41419034004211425,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 1.176491618156433,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.4723679542541504,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.5706663727760315,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.4028864860534668,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.5537962317466736,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.3607978820800781,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.6196736097335815,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.42556142807006836,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6695680022239685,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.41333789825439454,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.655034065246582,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5096014976501465,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.5938177704811096,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5216481685638428,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.7804147601127625,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.43359966278076173,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.7048928141593933,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.4395446300506592,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6047908067703247,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.4992673397064209,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6019694209098816,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.41515302658081055,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.8708873391151428,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.4704007625579834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6083199381828308,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5111989974975586,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5877307653427124,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48946352005004884,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6169144511222839,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39699459075927734,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7712729573249817,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.43967700004577637,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.7823882102966309,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.39926810264587403,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6713236570358276,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.454238748550415,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.8165543079376221,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.46351184844970705,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.37962615489959717,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.4180287837982178,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.6618972420692444,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.3922520637512207,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.42513608932495117,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46528077125549316,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5760522484779358,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4353321075439453,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.7091104984283447,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4817354679107666,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.7984012365341187,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.45444478988647463,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7693517804145813,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.4780622959136963,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5142762660980225,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4113470554351807,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.682898223400116,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.48437914848327634,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.3431150913238525,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.5317914962768555,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.7310293316841125,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.41095829010009766,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.564328134059906,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.4005306243896484,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6681087017059326,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.399173641204834,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.5092662572860718,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3901823043823242,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.0772058963775635,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4496616840362549,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.596607506275177,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.43782577514648435,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6527637243270874,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4539346218109131,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.6109674572944641,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.4309091091156006,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.4367820918560028,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.4418576717376709,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6753432154655457,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5508285522460937,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.605097234249115,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.47382965087890627,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.4321838617324829,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.3830132961273193,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6553069353103638,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.4628459930419922,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5756787657737732,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42679872512817385,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5278208255767822,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.39625184535980223,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.7524205446243286,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4828816890716553,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.7251951694488525,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.4799030780792236,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 1.0915361642837524,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4493619441986084,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.586887538433075,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.4356196403503418,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.601038932800293,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4816599369049072,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.5943931937217712,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.43202710151672363,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7597604393959045,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.47631521224975587,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.9090297222137451,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.43398633003234866,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.9506173729896545,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.4622031211853027,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.6177131533622742,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.37497625350952146,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.9170241951942444,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.4830442428588867,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.5221230983734131,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.4639876842498779,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.6676084995269775,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.36029102802276614,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.41131699085235596,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.4241457939147949,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.475800096988678,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.35529179573059083,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6368981003761292,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.38230233192443847,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.5252333879470825,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6526734352111816,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0308051109313965,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.4116359233856201,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.5103258490562439,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3934061288833618,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.2025281190872192,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.43854265213012694,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.49516773223876953,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.40279831886291506,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.631552517414093,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.4471002101898193,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.9680405259132385,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.3953091621398926,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6364196538925171,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.37395734786987306,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.7081876397132874,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.46148090362548827,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.5005178451538086,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43246755599975584,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.8044948577880859,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5013750553131103,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 1.4904066324234009,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.42342548370361327,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.40624552965164185,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.41327810287475586,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.9344062805175781,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.40997905731201173,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.9273070693016052,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4784945011138916,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.9853499531745911,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.41073040962219237,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 1.8092626333236694,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4564663410186768,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.2153793573379517,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.431945276260376,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.8164740204811096,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.4268978118896484,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.665752112865448,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.46366491317749026,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5824525952339172,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7948373794555664,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.5864537358283997,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4185792922973633,
+      "step": 600
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.253076758011774e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..122b904336253511a683844b10d46309d143b784
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-200/trainer_state.json
@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.16570008285004142,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.7295929789543152,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.035770255327224734,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.581419050693512,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.025004851818084716,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.20224618911743164,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.025373575091362,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.2050512433052063,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.027632910013198852,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.38691437244415283,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.031137889623641966,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 0.7509753704071045,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.04050299525260925,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.3284788429737091,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.03967776894569397,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.6000391244888306,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.03806843757629395,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.30676934123039246,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.041826143860816956,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.5363546013832092,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.018714608252048494,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.47426751255989075,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.029283612966537476,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.24491599202156067,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.030531010031700133,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.2878497540950775,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.02707911729812622,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.22228732705116272,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.01117827296257019,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.8952307105064392,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.02729830741882324,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.5875809788703918,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.04196040332317352,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.20279905200004578,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.05431713461875916,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.22159680724143982,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.039883467555046084,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.4569237232208252,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.05425180196762085,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.8258567452430725,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.036851489543914796,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7035917043685913,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.05425626039505005,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.40458786487579346,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.05570248961448669,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.2876298725605011,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.05134947299957275,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.41039037704467773,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.03166348338127136,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8085739016532898,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.08710886836051941,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.3362036943435669,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.06054364442825318,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.3655257225036621,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.04392111599445343,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6922850608825684,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.06741544008255004,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6054010391235352,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.02633320689201355,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 1.1901980638504028,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.0946669340133667,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8374701142311096,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.069843989610672,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.662906289100647,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.06582951545715332,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7055565714836121,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.04184426963329315,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 2.3092219829559326,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.06460075974464416,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.7795400619506836,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.06587035059928895,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 1.4828706979751587,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.026828449964523316,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.1976030468940735,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.06440910100936889,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.571037232875824,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.0584525465965271,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.3333803415298462,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.03820371925830841,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.11696146428585052,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.020550067722797393,
+      "step": 200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.182661353856339e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c69b014bf0d261393f0ac5d3e70dc26a8fb4325
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkonly/checkpoint-400/trainer_state.json
@@ -0,0 +1,594 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.33140016570008285,
+  "eval_steps": 500,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.7295929789543152,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.035770255327224734,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.581419050693512,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.025004851818084716,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.20224618911743164,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.025373575091362,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.2050512433052063,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.027632910013198852,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.38691437244415283,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.031137889623641966,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 0.7509753704071045,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.04050299525260925,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.3284788429737091,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.03967776894569397,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.6000391244888306,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.03806843757629395,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.30676934123039246,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.041826143860816956,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.5363546013832092,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.018714608252048494,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.47426751255989075,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.029283612966537476,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.24491599202156067,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.030531010031700133,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.2878497540950775,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.02707911729812622,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.22228732705116272,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.01117827296257019,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.8952307105064392,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.02729830741882324,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.5875809788703918,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.04196040332317352,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.20279905200004578,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.05431713461875916,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.22159680724143982,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.039883467555046084,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.4569237232208252,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.05425180196762085,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.8258567452430725,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.036851489543914796,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7035917043685913,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.05425626039505005,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.40458786487579346,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.05570248961448669,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.2876298725605011,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.05134947299957275,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.41039037704467773,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.03166348338127136,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8085739016532898,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.08710886836051941,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.3362036943435669,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.06054364442825318,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.3655257225036621,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.04392111599445343,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6922850608825684,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.06741544008255004,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6054010391235352,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.02633320689201355,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 1.1901980638504028,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.0946669340133667,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8374701142311096,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.069843989610672,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.662906289100647,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.06582951545715332,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7055565714836121,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.04184426963329315,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 2.3092219829559326,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.06460075974464416,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.7795400619506836,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.06587035059928895,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 1.4828706979751587,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.026828449964523316,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.1976030468940735,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.06440910100936889,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.571037232875824,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.0584525465965271,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.3333803415298462,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.03820371925830841,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.11696146428585052,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.020550067722797393,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.34504398703575134,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.015381921827793122,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5985000729560852,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.04327248632907867,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.4532170593738556,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.02830135226249695,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.605568528175354,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.05404549241065979,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.509525716304779,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.05440375208854675,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.7568187117576599,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.05271175503730774,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.3611878454685211,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.04962030351161957,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.3583345413208008,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.05039881467819214,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.2776690721511841,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.055028438568115234,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.548402726650238,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.045180055499076846,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.7795302867889404,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.06223986744880676,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.4252273738384247,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.06269790530204773,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.7764860391616821,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.028677907586097718,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.6435965299606323,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.055135542154312135,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 1.4476184844970703,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.042802608013153075,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6738976240158081,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.058124834299087526,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5723819732666016,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.05005825161933899,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.20089659094810486,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.03145417273044586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.48794716596603394,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.02934512495994568,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.5058144330978394,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.05772849917411804,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.37383031845092773,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.05983699560165405,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.4393402338027954,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.0481719434261322,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.5560103058815002,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.05901755094528198,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.9613159894943237,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.08065288662910461,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.9213653802871704,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.05289219617843628,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.7945060133934021,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.06129953861236572,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.7128658890724182,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.06524198651313781,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.44649913907051086,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.03650871813297272,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.3071570098400116,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.02829044759273529,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.45135122537612915,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.03217353522777557,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.41901424527168274,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.02634226679801941,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.2472295761108398,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.05802941918373108,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.34984642267227173,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.042967066168785095,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.9867066740989685,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.03153141736984253,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.6169705390930176,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.045670893788337705,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.08094340562820435,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.04038102924823761,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.49996358156204224,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.05917595624923706,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.37578192353248596,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.03594317138195038,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.12884461879730225,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.050359851121902464,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.5507317781448364,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.05471110939979553,
+      "step": 400
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.375348010692678e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/all_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..06818f942315228ea2d65b67430b6a6bc2be270d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/all_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.532391257768261e+17,
+    "train_loss": 0.3342979509374181,
+    "train_runtime": 23020.5505,
+    "train_samples_per_second": 0.052,
+    "train_steps_per_second": 0.052
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..89a646c3fcaddad4676e43e5ffe7b7e0d833f703
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1000/trainer_state.json
@@ -0,0 +1,1434 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8285004142502072,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5087592005729675,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.29485764503479006,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5704765319824219,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.33067688941955564,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.6747835278511047,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.30224220752716063,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6387521624565125,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.28526697158813474,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.4535050094127655,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.3205925226211548,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.6089757084846497,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.33173117637634275,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.5039451122283936,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.3561999320983887,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.5600525736808777,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.3447849750518799,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.4545106887817383,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.3560582160949707,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6351876258850098,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.33849263191223145,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.6935901641845703,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.36296839714050294,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5181876420974731,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.37508158683776854,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.3825652301311493,
+      "learning_rate": 5e-05,
+      "loss": 0.2728895664215088,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7595435976982117,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.34779109954833987,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4101616442203522,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.28774254322052,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.5258774757385254,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.36954126358032224,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.466656893491745,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.37073726654052735,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5617334842681885,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.3355105400085449,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.41381868720054626,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.3170324325561523,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7313498854637146,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.34540324211120604,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.47589200735092163,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.2623107433319092,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.3540254533290863,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.2762061595916748,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.47523629665374756,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.383648157119751,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.23593008518219,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3139298677444458,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5242756009101868,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.29867799282073976,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.0130422115325928,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.3410639762878418,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5193675756454468,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.3559127330780029,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5491055250167847,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.2827305316925049,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5315759778022766,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.34628896713256835,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.758928656578064,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.3520486831665039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.43907085061073303,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.27119152545928954,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5632044672966003,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.35201799869537354,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.2445769309997559,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.402988862991333,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.2879709303379059,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.25113420486450194,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5718819499015808,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.34347071647644045,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.6167446970939636,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.33843297958374025,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.613071620464325,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.6379887580871582,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5702184438705444,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3123698472976685,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.6785452961921692,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.32242157459259035,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5149484872817993,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.36657283306121824,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6781711578369141,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.28925161361694335,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.849950909614563,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.4696540355682373,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.7415074706077576,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.29894137382507324,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4377768635749817,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.3466787815093994,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.204514980316162,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.28234999179840087,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.7100546956062317,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.3697226047515869,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.8547986745834351,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.36382997035980225,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4771670401096344,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.3696743011474609,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.4570496380329132,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.2579970359802246,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.5765849351882935,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.2797149896621704,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5157090425491333,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.3318255662918091,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3839966058731079,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.34926605224609375,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7974236607551575,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.3199951171875,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.4744097888469696,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.29803919792175293,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.680403470993042,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3129017114639282,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6118494868278503,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.33185343742370604,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.2903285622596741,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.26662919521331785,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.41904616355895996,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.3488197088241577,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.4118332862854004,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.38364784717559813,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.47493502497673035,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.31982975006103515,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.5887065529823303,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.35746288299560547,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.3448180556297302,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.325803279876709,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.37045276165008545,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.2930459022521973,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6390769481658936,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.2991382360458374,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.49119478464126587,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3099710941314697,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5186780095100403,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.3839128017425537,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.46426114439964294,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.2749185562133789,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6482223868370056,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.33083789348602294,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.39385688304901123,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.30375258922576903,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.4826224148273468,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.5418938636779785,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.1159311532974243,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3104912281036377,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5817947387695312,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.6208138942718506,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.5510234236717224,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.3207552909851074,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.6656430959701538,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.3465660810470581,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.373602032661438,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.32450270652770996,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.38572436571121216,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.31318962574005127,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.48069295287132263,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.36074349880218504,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.4203653633594513,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.32322165966033933,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.5523374676704407,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.3633419513702393,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.810015082359314,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3288354158401489,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0981874800369683e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..645d708fd7a1c8904e6d3b97d9d425750a485bf2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1200/trainer_state.json
@@ -0,0 +1,1714 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9942004971002486,
+  "eval_steps": 500,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5087592005729675,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.29485764503479006,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5704765319824219,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.33067688941955564,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.6747835278511047,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.30224220752716063,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6387521624565125,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.28526697158813474,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.4535050094127655,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.3205925226211548,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.6089757084846497,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.33173117637634275,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.5039451122283936,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.3561999320983887,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.5600525736808777,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.3447849750518799,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.4545106887817383,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.3560582160949707,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6351876258850098,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.33849263191223145,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.6935901641845703,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.36296839714050294,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5181876420974731,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.37508158683776854,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.3825652301311493,
+      "learning_rate": 5e-05,
+      "loss": 0.2728895664215088,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7595435976982117,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.34779109954833987,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4101616442203522,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.28774254322052,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.5258774757385254,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.36954126358032224,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.466656893491745,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.37073726654052735,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5617334842681885,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.3355105400085449,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.41381868720054626,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.3170324325561523,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7313498854637146,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.34540324211120604,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.47589200735092163,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.2623107433319092,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.3540254533290863,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.2762061595916748,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.47523629665374756,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.383648157119751,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.23593008518219,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3139298677444458,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5242756009101868,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.29867799282073976,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.0130422115325928,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.3410639762878418,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5193675756454468,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.3559127330780029,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5491055250167847,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.2827305316925049,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5315759778022766,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.34628896713256835,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.758928656578064,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.3520486831665039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.43907085061073303,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.27119152545928954,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5632044672966003,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.35201799869537354,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.2445769309997559,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.402988862991333,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.2879709303379059,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.25113420486450194,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5718819499015808,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.34347071647644045,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.6167446970939636,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.33843297958374025,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.613071620464325,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.6379887580871582,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5702184438705444,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3123698472976685,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.6785452961921692,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.32242157459259035,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5149484872817993,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.36657283306121824,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6781711578369141,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.28925161361694335,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.849950909614563,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.4696540355682373,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.7415074706077576,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.29894137382507324,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4377768635749817,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.3466787815093994,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.204514980316162,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.28234999179840087,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.7100546956062317,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.3697226047515869,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.8547986745834351,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.36382997035980225,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4771670401096344,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.3696743011474609,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.4570496380329132,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.2579970359802246,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.5765849351882935,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.2797149896621704,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5157090425491333,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.3318255662918091,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3839966058731079,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.34926605224609375,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7974236607551575,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.3199951171875,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.4744097888469696,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.29803919792175293,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.680403470993042,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3129017114639282,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6118494868278503,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.33185343742370604,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.2903285622596741,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.26662919521331785,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.41904616355895996,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.3488197088241577,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.4118332862854004,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.38364784717559813,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.47493502497673035,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.31982975006103515,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.5887065529823303,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.35746288299560547,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.3448180556297302,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.325803279876709,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.37045276165008545,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.2930459022521973,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6390769481658936,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.2991382360458374,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.49119478464126587,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3099710941314697,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5186780095100403,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.3839128017425537,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.46426114439964294,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.2749185562133789,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6482223868370056,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.33083789348602294,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.39385688304901123,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.30375258922576903,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.4826224148273468,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.5418938636779785,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.1159311532974243,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3104912281036377,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5817947387695312,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.6208138942718506,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.5510234236717224,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.3207552909851074,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.6656430959701538,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.3465660810470581,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.373602032661438,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.32450270652770996,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.38572436571121216,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.31318962574005127,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.48069295287132263,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.36074349880218504,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.4203653633594513,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.32322165966033933,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.5523374676704407,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.3633419513702393,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.810015082359314,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3288354158401489,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.8565049767494202,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.34276857376098635,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.5572496056556702,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.36288676261901853,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.5529004335403442,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.26964316368103025,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8306896686553955,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.40260953903198243,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4062662422657013,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.3201975107192993,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6771445870399475,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.37618632316589357,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.32105356454849243,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3110248327255249,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.35568806529045105,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.36226863861083985,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.8121322989463806,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.3662156820297241,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.4652898609638214,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.345924973487854,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.3441062867641449,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.3228020429611206,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.5775281190872192,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.3671683073043823,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.44707056879997253,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.3490464687347412,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.3276211619377136,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.3606743097305298,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.31616097688674927,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.3412326812744141,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.4623435139656067,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.3896538257598877,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5683387517929077,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.2929984569549561,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1409229040145874,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.37584359645843507,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.7672995924949646,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.28886773586273196,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.4839645326137543,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.3259925365447998,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.488469660282135,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.3495216369628906,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7248877882957458,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3109825611114502,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.4968997538089752,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.32770657539367676,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5228226184844971,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.32598109245300294,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.48649173974990845,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.31155178546905515,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.4252619743347168,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.2973174571990967,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6076405644416809,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.32831621170043945,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8747165203094482,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.366020655632019,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.687702476978302,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.3503147602081299,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6303551197052002,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3065986633300781,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7641124129295349,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.35827012062072755,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5009490847587585,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.2859072208404541,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.4684002697467804,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.34689414501190186,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.36474862694740295,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.295949649810791,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.43962275981903076,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.2861870050430298,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6348811984062195,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.3552159547805786,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.5709355473518372,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.288747763633728,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.47271692752838135,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.29303388595581054,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.37458208203315735,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.3211140871047974,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.878754198551178,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.37377498149871824,
+      "step": 1200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5182573450197856e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..48d9d2b8d5e023ea85ff46b7fe98bcd5485267e4
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207/trainer_state.json
@@ -0,0 +1,1721 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5087592005729675,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.29485764503479006,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5704765319824219,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.33067688941955564,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.6747835278511047,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.30224220752716063,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6387521624565125,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.28526697158813474,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.4535050094127655,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.3205925226211548,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.6089757084846497,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.33173117637634275,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.5039451122283936,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.3561999320983887,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.5600525736808777,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.3447849750518799,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.4545106887817383,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.3560582160949707,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6351876258850098,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.33849263191223145,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.6935901641845703,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.36296839714050294,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5181876420974731,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.37508158683776854,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.3825652301311493,
+      "learning_rate": 5e-05,
+      "loss": 0.2728895664215088,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7595435976982117,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.34779109954833987,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4101616442203522,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.28774254322052,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.5258774757385254,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.36954126358032224,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.466656893491745,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.37073726654052735,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5617334842681885,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.3355105400085449,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.41381868720054626,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.3170324325561523,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7313498854637146,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.34540324211120604,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.47589200735092163,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.2623107433319092,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.3540254533290863,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.2762061595916748,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.47523629665374756,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.383648157119751,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.23593008518219,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3139298677444458,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5242756009101868,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.29867799282073976,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.0130422115325928,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.3410639762878418,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5193675756454468,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.3559127330780029,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5491055250167847,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.2827305316925049,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5315759778022766,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.34628896713256835,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.758928656578064,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.3520486831665039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.43907085061073303,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.27119152545928954,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5632044672966003,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.35201799869537354,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.2445769309997559,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.402988862991333,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.2879709303379059,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.25113420486450194,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5718819499015808,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.34347071647644045,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.6167446970939636,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.33843297958374025,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.613071620464325,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.6379887580871582,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5702184438705444,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3123698472976685,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.6785452961921692,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.32242157459259035,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5149484872817993,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.36657283306121824,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6781711578369141,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.28925161361694335,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.849950909614563,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.4696540355682373,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.7415074706077576,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.29894137382507324,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4377768635749817,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.3466787815093994,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.204514980316162,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.28234999179840087,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.7100546956062317,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.3697226047515869,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.8547986745834351,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.36382997035980225,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4771670401096344,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.3696743011474609,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.4570496380329132,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.2579970359802246,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.5765849351882935,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.2797149896621704,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5157090425491333,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.3318255662918091,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3839966058731079,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.34926605224609375,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7974236607551575,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.3199951171875,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.4744097888469696,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.29803919792175293,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.680403470993042,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3129017114639282,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6118494868278503,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.33185343742370604,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.2903285622596741,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.26662919521331785,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.41904616355895996,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.3488197088241577,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.4118332862854004,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.38364784717559813,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.47493502497673035,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.31982975006103515,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.5887065529823303,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.35746288299560547,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.3448180556297302,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.325803279876709,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.37045276165008545,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.2930459022521973,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6390769481658936,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.2991382360458374,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.49119478464126587,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3099710941314697,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5186780095100403,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.3839128017425537,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.46426114439964294,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.2749185562133789,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6482223868370056,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.33083789348602294,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.39385688304901123,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.30375258922576903,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.4826224148273468,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.5418938636779785,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.1159311532974243,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3104912281036377,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5817947387695312,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.6208138942718506,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.5510234236717224,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.3207552909851074,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.6656430959701538,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.3465660810470581,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.373602032661438,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.32450270652770996,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.38572436571121216,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.31318962574005127,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.48069295287132263,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.36074349880218504,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.4203653633594513,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.32322165966033933,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.5523374676704407,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.3633419513702393,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.810015082359314,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3288354158401489,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.8565049767494202,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.34276857376098635,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.5572496056556702,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.36288676261901853,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.5529004335403442,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.26964316368103025,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8306896686553955,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.40260953903198243,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4062662422657013,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.3201975107192993,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6771445870399475,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.37618632316589357,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.32105356454849243,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3110248327255249,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.35568806529045105,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.36226863861083985,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.8121322989463806,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.3662156820297241,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.4652898609638214,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.345924973487854,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.3441062867641449,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.3228020429611206,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.5775281190872192,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.3671683073043823,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.44707056879997253,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.3490464687347412,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.3276211619377136,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.3606743097305298,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.31616097688674927,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.3412326812744141,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.4623435139656067,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.3896538257598877,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5683387517929077,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.2929984569549561,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1409229040145874,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.37584359645843507,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.7672995924949646,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.28886773586273196,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.4839645326137543,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.3259925365447998,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.488469660282135,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.3495216369628906,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7248877882957458,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3109825611114502,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.4968997538089752,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.32770657539367676,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5228226184844971,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.32598109245300294,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.48649173974990845,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.31155178546905515,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.4252619743347168,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.2973174571990967,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6076405644416809,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.32831621170043945,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8747165203094482,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.366020655632019,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.687702476978302,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.3503147602081299,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6303551197052002,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3065986633300781,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7641124129295349,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.35827012062072755,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5009490847587585,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.2859072208404541,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.4684002697467804,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.34689414501190186,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.36474862694740295,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.295949649810791,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.43962275981903076,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.2861870050430298,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6348811984062195,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.3552159547805786,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.5709355473518372,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.288747763633728,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.47271692752838135,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.29303388595581054,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.37458208203315735,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.3211140871047974,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.878754198551178,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.37377498149871824,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 0.9046036005020142,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.286467456817627,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532391257768261e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..df32e6e6d2fab3e96f8c832fd2e2ccd5c78a99a7
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-200/trainer_state.json
@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.16570008285004142,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.182661353856339e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f2153e678edc6ccbdd3a8202e7625de44731334
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-400/trainer_state.json
@@ -0,0 +1,594 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.33140016570008285,
+  "eval_steps": 500,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.375348010692678e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..804046887efa6f1707ad92764823c61f91a83aad
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-600/trainer_state.json
@@ -0,0 +1,874 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.4971002485501243,
+  "eval_steps": 500,
+  "global_step": 600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2530762997122093e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..98b9a0c8de175009d3790a31d2430328786c0578
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-800/trainer_state.json
@@ -0,0 +1,1154 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6628003314001657,
+  "eval_steps": 500,
+  "global_step": 800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5087592005729675,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.29485764503479006,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5704765319824219,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.33067688941955564,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.6747835278511047,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.30224220752716063,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6387521624565125,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.28526697158813474,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.4535050094127655,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.3205925226211548,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.6089757084846497,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.33173117637634275,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.5039451122283936,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.3561999320983887,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.5600525736808777,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.3447849750518799,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.4545106887817383,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.3560582160949707,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6351876258850098,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.33849263191223145,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.6935901641845703,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.36296839714050294,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5181876420974731,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.37508158683776854,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.3825652301311493,
+      "learning_rate": 5e-05,
+      "loss": 0.2728895664215088,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7595435976982117,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.34779109954833987,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4101616442203522,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.28774254322052,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.5258774757385254,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.36954126358032224,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.466656893491745,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.37073726654052735,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5617334842681885,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.3355105400085449,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.41381868720054626,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.3170324325561523,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7313498854637146,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.34540324211120604,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.47589200735092163,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.2623107433319092,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.3540254533290863,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.2762061595916748,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.47523629665374756,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.383648157119751,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.23593008518219,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3139298677444458,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5242756009101868,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.29867799282073976,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.0130422115325928,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.3410639762878418,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5193675756454468,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.3559127330780029,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5491055250167847,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.2827305316925049,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5315759778022766,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.34628896713256835,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.758928656578064,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.3520486831665039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.43907085061073303,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.27119152545928954,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5632044672966003,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.35201799869537354,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.2445769309997559,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.402988862991333,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.2879709303379059,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.25113420486450194,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5718819499015808,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.34347071647644045,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.6167446970939636,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.33843297958374025,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.613071620464325,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.6379887580871582,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5702184438705444,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3123698472976685,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.6785452961921692,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.32242157459259035,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5149484872817993,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.36657283306121824,
+      "step": 800
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6775645391393504e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/model.safetensors b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..000bb92e32e530eea4e37a808a27ca3add3dd276
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f1eaeab665a8d40acdc391c43cb44556cc61b68479f5a7adc2b4ab5142ff41d
+size 1867938664
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ac718f112af542c39adcd73bb3bde829991ce0c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf03774334462d6e34b5005cb11120a62275f146ee2953e68731ecdbce84fbb
+size 11423222
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/train_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..06818f942315228ea2d65b67430b6a6bc2be270d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.532391257768261e+17,
+    "train_loss": 0.3342979509374181,
+    "train_runtime": 23020.5505,
+    "train_samples_per_second": 0.052,
+    "train_steps_per_second": 0.052
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ffaa1e55d7d7355adcf8ac65d7bb857d5afd0bd
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/trainer_state.json
@@ -0,0 +1,1730 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.977551281452179,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 0.39611239433288575,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.44359347224235535,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.28547301292419436,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.42553624510765076,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.38471646308898927,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.5327292680740356,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.2959540367126465,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.40513092279434204,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.35033435821533204,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 1.0036418437957764,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.3590218544006348,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.40394946932792664,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.35713920593261717,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.7791556715965271,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.36656599044799804,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.3966231942176819,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.3739785194396973,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7675734162330627,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.3125196933746338,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5979276299476624,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.31746151447296145,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4086473882198334,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.3270680904388428,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.36814939975738525,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.295710825920105,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.4568744897842407,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.24994416236877443,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.698323130607605,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.2805319309234619,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.7690544128417969,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.3356164455413818,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.4944077134132385,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.3348794937133789,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.4026140570640564,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.585720157623291,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.9295781850814819,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.3494632482528687,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.6363644599914551,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.36145284175872805,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.7371365427970886,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.3258823394775391,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.5603595972061157,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.3408616542816162,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.769363522529602,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.311154842376709,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.7764161825180054,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.30724825859069826,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.936690092086792,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.348107123374939,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5967389345169067,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.3580388307571411,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.4234471917152405,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.30428709983825686,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.6024398803710938,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.3735044002532959,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.6135967969894409,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.2945706367492676,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.82120680809021,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.3924151659011841,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.8878151178359985,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.35369362831115725,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.6515068411827087,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.38618485927581786,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.5059307813644409,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.31272971630096436,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.766676902770996,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.43032150268554686,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6427023410797119,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.32725460529327394,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.5139245986938477,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.2663165330886841,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.5557900667190552,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.33349170684814455,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.4914829730987549,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.30496530532836913,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.6873228549957275,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.3162576198577881,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.49221423268318176,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.27688000202178953,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.45048314332962036,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.23724915981292724,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.5706700682640076,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.30625813007354735,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6094169616699219,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.2834159851074219,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.5847861766815186,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.3459680795669556,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.44215551018714905,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.37563333511352537,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.6549540162086487,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.31489880084991456,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.5488448143005371,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.33098340034484863,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4822295010089874,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.361669135093689,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.8312385678291321,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.29588654041290285,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.678106427192688,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.31206045150756834,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.685511589050293,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.359794020652771,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5240558981895447,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.36330993175506593,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6167172193527222,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.2819963932037354,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.8909366726875305,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.33033390045166017,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.8025505542755127,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.2827253818511963,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.5908136367797852,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.3376539945602417,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.5588940382003784,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.3264751672744751,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3343297839164734,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.29050137996673586,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.5140760540962219,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.2783830642700195,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.39502424001693726,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.3482329845428467,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5052193403244019,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.33042938709259034,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.534315824508667,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.34135894775390624,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.739052414894104,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.3434577465057373,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.7862807512283325,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.37205140590667723,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.5172637104988098,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.2968695402145386,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.6243228316307068,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.37090587615966797,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.9181405305862427,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.387163782119751,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.5545716881752014,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.286794638633728,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.5396609902381897,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.2775604486465454,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6781189441680908,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.28359901905059814,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.4333276152610779,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.27118003368377686,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.847393810749054,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.3385643005371094,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4992876648902893,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.31628530025482177,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.6148054003715515,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.3191103458404541,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.7377797961235046,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.319942569732666,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.41700002551078796,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.33182401657104493,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6291422843933105,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.39645678997039796,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.5585975050926208,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.3358659029006958,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.3406830132007599,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.2914732933044434,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6737725734710693,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.3541771173477173,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5509101748466492,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.3073173999786377,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 1.1471089124679565,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.286766505241394,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.4982381761074066,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.35014164447784424,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6030893921852112,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.3514642953872681,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6634058356285095,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.33918418884277346,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.5166270732879639,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.32272164821624755,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4971492886543274,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.3475889921188354,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.4489460587501526,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.3374284029006958,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 1.1264560222625732,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.3569037914276123,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7206049561500549,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.3293704271316528,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8477029204368591,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.3469815254211426,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3657445013523102,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.2762319326400757,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4479973614215851,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.34645495414733884,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.4124138653278351,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.35814087390899657,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.7036811113357544,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.26125392913818357,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.3255551755428314,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.3123605251312256,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.3617493510246277,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.2636122465133667,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.41319799423217773,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.27571537494659426,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.9957762956619263,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.4861611843109131,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 1.0447680950164795,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.29194035530090334,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.32469314336776733,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.2917078733444214,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.1182771921157837,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.326418399810791,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.42194414138793945,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.3099753618240356,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5116533637046814,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.3342925786972046,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5804761052131653,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.2874398708343506,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.6405546069145203,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.27805142402648925,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6012183427810669,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.3499223470687866,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.4284619390964508,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.33189918994903567,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.6063693165779114,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.39174885749816896,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.5874237418174744,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.321898889541626,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3535780608654022,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.322508716583252,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.34793734550476074,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.3147265911102295,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5320057272911072,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.361543607711792,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 1.0222864151000977,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.3204171180725098,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.5175772309303284,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.33651018142700195,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 1.684689998626709,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.3201303005218506,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5160192847251892,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.30648729801177976,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.34148502349853516,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.35619707107543946,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5264866352081299,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.6279504776000977,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43867921829223633,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.3109663248062134,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5087592005729675,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.29485764503479006,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5704765319824219,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.33067688941955564,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.6747835278511047,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.30224220752716063,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6387521624565125,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.28526697158813474,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.4535050094127655,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.3205925226211548,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.6089757084846497,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.33173117637634275,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.5039451122283936,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.3561999320983887,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.5600525736808777,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.3447849750518799,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.4545106887817383,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.3560582160949707,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6351876258850098,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.33849263191223145,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.6935901641845703,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.36296839714050294,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5181876420974731,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.37508158683776854,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.3825652301311493,
+      "learning_rate": 5e-05,
+      "loss": 0.2728895664215088,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.7595435976982117,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.34779109954833987,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4101616442203522,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.28774254322052,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.5258774757385254,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.36954126358032224,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.466656893491745,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.37073726654052735,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5617334842681885,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.3355105400085449,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.41381868720054626,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.3170324325561523,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7313498854637146,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.34540324211120604,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.47589200735092163,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.2623107433319092,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.3540254533290863,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.2762061595916748,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.47523629665374756,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.383648157119751,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.23593008518219,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3139298677444458,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5242756009101868,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.29867799282073976,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 1.0130422115325928,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.3410639762878418,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5193675756454468,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.3559127330780029,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5491055250167847,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.2827305316925049,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5315759778022766,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.34628896713256835,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.758928656578064,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.3520486831665039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.43907085061073303,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.27119152545928954,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5632044672966003,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.35201799869537354,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 1.2445769309997559,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.402988862991333,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.2879709303379059,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.25113420486450194,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5718819499015808,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.34347071647644045,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.6167446970939636,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.33843297958374025,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.613071620464325,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.6379887580871582,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5702184438705444,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.3123698472976685,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.6785452961921692,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.32242157459259035,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5149484872817993,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.36657283306121824,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6781711578369141,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.28925161361694335,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.849950909614563,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.4696540355682373,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.7415074706077576,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.29894137382507324,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.4377768635749817,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.3466787815093994,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 1.204514980316162,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.28234999179840087,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.7100546956062317,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.3697226047515869,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.8547986745834351,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.36382997035980225,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4771670401096344,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.3696743011474609,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.4570496380329132,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.2579970359802246,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.5765849351882935,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.2797149896621704,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5157090425491333,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.3318255662918091,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3839966058731079,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.34926605224609375,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7974236607551575,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.3199951171875,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.4744097888469696,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.29803919792175293,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.680403470993042,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.3129017114639282,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6118494868278503,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.33185343742370604,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.2903285622596741,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.26662919521331785,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.41904616355895996,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.3488197088241577,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.4118332862854004,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.38364784717559813,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.47493502497673035,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.31982975006103515,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.5887065529823303,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.35746288299560547,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.3448180556297302,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.325803279876709,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.37045276165008545,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.2930459022521973,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6390769481658936,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.2991382360458374,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.49119478464126587,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.3099710941314697,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5186780095100403,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.3839128017425537,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.46426114439964294,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.2749185562133789,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6482223868370056,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.33083789348602294,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.39385688304901123,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.30375258922576903,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.4826224148273468,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.5418938636779785,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 1.1159311532974243,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3104912281036377,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.5817947387695312,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.6208138942718506,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.5510234236717224,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.3207552909851074,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.6656430959701538,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.3465660810470581,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.373602032661438,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.32450270652770996,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.38572436571121216,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.31318962574005127,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.48069295287132263,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.36074349880218504,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.4203653633594513,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.32322165966033933,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.5523374676704407,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.3633419513702393,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.810015082359314,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.3288354158401489,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.8565049767494202,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.34276857376098635,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.5572496056556702,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.36288676261901853,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.5529004335403442,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.26964316368103025,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.8306896686553955,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.40260953903198243,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.4062662422657013,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.3201975107192993,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.6771445870399475,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.37618632316589357,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.32105356454849243,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3110248327255249,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.35568806529045105,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.36226863861083985,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.8121322989463806,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.3662156820297241,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.4652898609638214,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.345924973487854,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.3441062867641449,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.3228020429611206,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.5775281190872192,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.3671683073043823,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.44707056879997253,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.3490464687347412,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.3276211619377136,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.3606743097305298,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.31616097688674927,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.3412326812744141,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.4623435139656067,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.3896538257598877,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.5683387517929077,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.2929984569549561,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 1.1409229040145874,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.37584359645843507,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.7672995924949646,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.28886773586273196,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.4839645326137543,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.3259925365447998,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.488469660282135,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.3495216369628906,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.7248877882957458,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.3109825611114502,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.4968997538089752,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.32770657539367676,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5228226184844971,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.32598109245300294,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.48649173974990845,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.31155178546905515,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.4252619743347168,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.2973174571990967,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6076405644416809,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.32831621170043945,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8747165203094482,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.366020655632019,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.687702476978302,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.3503147602081299,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6303551197052002,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.3065986633300781,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 0.7641124129295349,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.35827012062072755,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5009490847587585,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.2859072208404541,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.4684002697467804,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.34689414501190186,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.36474862694740295,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.295949649810791,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.43962275981903076,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.2861870050430298,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.6348811984062195,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.3552159547805786,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.5709355473518372,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.288747763633728,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.47271692752838135,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.29303388595581054,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.37458208203315735,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.3211140871047974,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 0.878754198551178,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.37377498149871824,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 0.9046036005020142,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.286467456817627,
+      "step": 1205
+    },
+    {
+      "epoch": 1.0,
+      "step": 1207,
+      "total_flos": 2.532391257768261e+17,
+      "train_loss": 0.3342979509374181,
+      "train_runtime": 23020.5505,
+      "train_samples_per_second": 0.052,
+      "train_steps_per_second": 0.052
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532391257768261e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/training_args.bin b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..9771318680e290f9aad48957455f211ee43d7c61
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a337dbe94a510a5d5b8b9fa0ab3122ea2f6809cec13ae6380a0917f813ab6af1
+size 5265
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/all_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..992babc967a3411672658e64a124bf778bf2e31c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/all_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.532391257768261e+17,
+    "train_loss": 0.35478373169405164,
+    "train_runtime": 41719.5034,
+    "train_samples_per_second": 0.029,
+    "train_steps_per_second": 0.029
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6febaf9670885d9e88916b73da4a5bbf938e5bfb
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e95c6fcdb83e6228d0fd8c74313697d9f44fd141
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.1"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/model.safetensors b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..632793bcde23c5c657dff0223d952f21886a36d0
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0ec7951180204a7e052f891ee3f23271f5a89b910494bec96c02527b0cc02c
+size 1867938664
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/optimizer.pt b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/optimizer.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68afe91fde37f4b84027a6195e9e090a1e474073
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c926a12925c41ba0f0eb1c880c7b56cf1bcb43f2431414bb6d95b870e0db6c3
+size 3734474915
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/rng_state.pth b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/rng_state.pth
new file mode 100644
index 0000000000000000000000000000000000000000..97ebb26b3341a5a2cd21ce1aecda77b009c3af68
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb00e232b3100956f7f65d8d227fc86d43cd89eb1020e69de20bc28c5cfad73
+size 14645
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/scheduler.pt b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b287ce342dad2ed16b84fd747e6010ada9605271
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:898c3bc7287ab4a5111299cf5c14412639549058ca27120a733bac291bff40ff
+size 1465
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ac718f112af542c39adcd73bb3bde829991ce0c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf03774334462d6e34b5005cb11120a62275f146ee2953e68731ecdbce84fbb
+size 11423222
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e95f3e4c32fd64d96a236e5a5d74a9ed99f72e17
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/trainer_state.json
@@ -0,0 +1,1721 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.9088397026062012,
+      "learning_rate": 3.3057851239669426e-07,
+      "loss": 0.46797571182250974,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.4194833040237427,
+      "learning_rate": 7.438016528925621e-07,
+      "loss": 0.32537107467651366,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.48602092266082764,
+      "learning_rate": 1.15702479338843e-06,
+      "loss": 0.440777587890625,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.615478515625,
+      "learning_rate": 1.5702479338842977e-06,
+      "loss": 0.3342358350753784,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.4429086148738861,
+      "learning_rate": 1.9834710743801654e-06,
+      "loss": 0.4068150043487549,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 0.7779952883720398,
+      "learning_rate": 2.3966942148760335e-06,
+      "loss": 0.4107870101928711,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.4232267439365387,
+      "learning_rate": 2.809917355371901e-06,
+      "loss": 0.4021468639373779,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.5329512357711792,
+      "learning_rate": 3.2231404958677685e-06,
+      "loss": 0.40447416305541994,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.40902742743492126,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.4133272647857666,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7031140327453613,
+      "learning_rate": 4.049586776859504e-06,
+      "loss": 0.3559228181838989,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5372095704078674,
+      "learning_rate": 4.462809917355372e-06,
+      "loss": 0.3345926761627197,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4524230360984802,
+      "learning_rate": 4.87603305785124e-06,
+      "loss": 0.3521779775619507,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.405799001455307,
+      "learning_rate": 5.289256198347108e-06,
+      "loss": 0.32509465217590333,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.6495310068130493,
+      "learning_rate": 5.702479338842976e-06,
+      "loss": 0.25270297527313235,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.565504252910614,
+      "learning_rate": 6.115702479338843e-06,
+      "loss": 0.2897182941436768,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.9850594401359558,
+      "learning_rate": 6.528925619834712e-06,
+      "loss": 0.35270593166351316,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.590770423412323,
+      "learning_rate": 6.942148760330579e-06,
+      "loss": 0.3310926198959351,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.3464272916316986,
+      "learning_rate": 7.355371900826447e-06,
+      "loss": 0.6234882831573486,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.6508628129959106,
+      "learning_rate": 7.768595041322314e-06,
+      "loss": 0.36177287101745603,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.4168076515197754,
+      "learning_rate": 8.181818181818183e-06,
+      "loss": 0.37993545532226564,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.4954686462879181,
+      "learning_rate": 8.59504132231405e-06,
+      "loss": 0.32210068702697753,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.3881250321865082,
+      "learning_rate": 9.008264462809918e-06,
+      "loss": 0.34254477024078367,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.4197976291179657,
+      "learning_rate": 9.421487603305785e-06,
+      "loss": 0.30668623447418214,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.4320375621318817,
+      "learning_rate": 9.834710743801654e-06,
+      "loss": 0.29941961765289304,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.4299139380455017,
+      "learning_rate": 9.999811713194481e-06,
+      "loss": 0.3240851402282715,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5377570986747742,
+      "learning_rate": 9.99866112295902e-06,
+      "loss": 0.34094393253326416,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.34123289585113525,
+      "learning_rate": 9.996464786686645e-06,
+      "loss": 0.28050880432128905,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.4427873492240906,
+      "learning_rate": 9.993223163862385e-06,
+      "loss": 0.34813799858093264,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.359440416097641,
+      "learning_rate": 9.988936932650682e-06,
+      "loss": 0.27970683574676514,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.5566115975379944,
+      "learning_rate": 9.98360698975351e-06,
+      "loss": 0.3594496250152588,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.4960085153579712,
+      "learning_rate": 9.977234450222783e-06,
+      "loss": 0.334060525894165,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.5455737709999084,
+      "learning_rate": 9.969820647227086e-06,
+      "loss": 0.3540655136108398,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.42919108271598816,
+      "learning_rate": 9.961367131772759e-06,
+      "loss": 0.29244580268859866,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.15133798122406,
+      "learning_rate": 9.951875672379424e-06,
+      "loss": 0.3785684108734131,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.3836948573589325,
+      "learning_rate": 9.941348254710004e-06,
+      "loss": 0.32334723472595217,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.38102057576179504,
+      "learning_rate": 9.929787081155305e-06,
+      "loss": 0.2559232711791992,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.4217875897884369,
+      "learning_rate": 9.91719457037327e-06,
+      "loss": 0.3103822708129883,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.43528667092323303,
+      "learning_rate": 9.903573356782978e-06,
+      "loss": 0.29477834701538086,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.7707864046096802,
+      "learning_rate": 9.888926290013521e-06,
+      "loss": 0.304648232460022,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.34778332710266113,
+      "learning_rate": 9.873256434307828e-06,
+      "loss": 0.27324905395507815,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.34213224053382874,
+      "learning_rate": 9.856567067881628e-06,
+      "loss": 0.22963452339172363,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.4351404011249542,
+      "learning_rate": 9.838861682237619e-06,
+      "loss": 0.28716602325439455,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.553987979888916,
+      "learning_rate": 9.820143981435031e-06,
+      "loss": 0.26800360679626467,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.49172210693359375,
+      "learning_rate": 9.800417881314717e-06,
+      "loss": 0.31684017181396484,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.36292967200279236,
+      "learning_rate": 9.779687508679936e-06,
+      "loss": 0.3675337791442871,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.5972428917884827,
+      "learning_rate": 9.757957200433011e-06,
+      "loss": 0.30315725803375243,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.6428586840629578,
+      "learning_rate": 9.73523150266802e-06,
+      "loss": 0.321216082572937,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4479713439941406,
+      "learning_rate": 9.711515169719732e-06,
+      "loss": 0.3546335458755493,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6334832906723022,
+      "learning_rate": 9.686813163168973e-06,
+      "loss": 0.26909546852111815,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.49879494309425354,
+      "learning_rate": 9.661130650804643e-06,
+      "loss": 0.269888710975647,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.4971435070037842,
+      "learning_rate": 9.634473005542579e-06,
+      "loss": 0.3372554063796997,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5778576135635376,
+      "learning_rate": 9.606845804301523e-06,
+      "loss": 0.3382244348526001,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.5813653469085693,
+      "learning_rate": 9.57825482683639e-06,
+      "loss": 0.2622408866882324,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.5511571764945984,
+      "learning_rate": 9.548706054529124e-06,
+      "loss": 0.3133584499359131,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.5167928338050842,
+      "learning_rate": 9.518205669137353e-06,
+      "loss": 0.24319915771484374,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.49302011728286743,
+      "learning_rate": 9.486760051501139e-06,
+      "loss": 0.3253422021865845,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.46096402406692505,
+      "learning_rate": 9.454375780208072e-06,
+      "loss": 0.3033352613449097,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.29232358932495117,
+      "learning_rate": 9.421059630216992e-06,
+      "loss": 0.28919391632080077,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.37558865547180176,
+      "learning_rate": 9.386818571440631e-06,
+      "loss": 0.2719090700149536,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.34535545110702515,
+      "learning_rate": 9.351659767287477e-06,
+      "loss": 0.32380578517913816,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.38860467076301575,
+      "learning_rate": 9.315590573163152e-06,
+      "loss": 0.32611970901489257,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.4789179861545563,
+      "learning_rate": 9.278618534931612e-06,
+      "loss": 0.31763691902160646,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.5553998947143555,
+      "learning_rate": 9.240751387336522e-06,
+      "loss": 0.3328229904174805,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.54727703332901,
+      "learning_rate": 9.201997052383107e-06,
+      "loss": 0.34824743270874026,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.6511346101760864,
+      "learning_rate": 9.162363637680822e-06,
+      "loss": 0.2723949432373047,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.5212080478668213,
+      "learning_rate": 9.121859434747205e-06,
+      "loss": 0.34482150077819823,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.8779405355453491,
+      "learning_rate": 9.080492917273238e-06,
+      "loss": 0.3662183046340942,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.8695563077926636,
+      "learning_rate": 9.038272739350617e-06,
+      "loss": 0.2668049097061157,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.4466971755027771,
+      "learning_rate": 8.995207733661266e-06,
+      "loss": 0.26048974990844725,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.4294697344303131,
+      "learning_rate": 8.951306909629492e-06,
+      "loss": 0.26141631603240967,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.38591551780700684,
+      "learning_rate": 8.906579451537172e-06,
+      "loss": 0.25841057300567627,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.6442238688468933,
+      "learning_rate": 8.861034716602343e-06,
+      "loss": 0.3345190525054932,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4142279028892517,
+      "learning_rate": 8.814682233021636e-06,
+      "loss": 0.2878074645996094,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.46708178520202637,
+      "learning_rate": 8.76753169797692e-06,
+      "loss": 0.2928623676300049,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 1.1409103870391846,
+      "learning_rate": 8.719592975606601e-06,
+      "loss": 0.2992920637130737,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.9347559213638306,
+      "learning_rate": 8.670876094941991e-06,
+      "loss": 0.33010694980621336,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.4922148585319519,
+      "learning_rate": 8.62139124780919e-06,
+      "loss": 0.3573878288269043,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.47879040241241455,
+      "learning_rate": 8.571148786696894e-06,
+      "loss": 0.29300789833068847,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.44266507029533386,
+      "learning_rate": 8.520159222590605e-06,
+      "loss": 0.27748234272003175,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.5809404850006104,
+      "learning_rate": 8.468433222773677e-06,
+      "loss": 0.3408142328262329,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.49024492502212524,
+      "learning_rate": 8.415981608595666e-06,
+      "loss": 0.291151237487793,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.40204304456710815,
+      "learning_rate": 8.362815353208441e-06,
+      "loss": 0.2749445915222168,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.5574970841407776,
+      "learning_rate": 8.308945579270551e-06,
+      "loss": 0.33534510135650636,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.7230126261711121,
+      "learning_rate": 8.2543835566203e-06,
+      "loss": 0.3234541416168213,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6410138010978699,
+      "learning_rate": 8.19914069991805e-06,
+      "loss": 0.323453688621521,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.689175009727478,
+      "learning_rate": 8.143228566258192e-06,
+      "loss": 0.29622983932495117,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4988920986652374,
+      "learning_rate": 8.08665885275137e-06,
+      "loss": 0.3264308929443359,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.5350947976112366,
+      "learning_rate": 8.029443394077356e-06,
+      "loss": 0.32159135341644285,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.5879245400428772,
+      "learning_rate": 7.971594160009195e-06,
+      "loss": 0.35001649856567385,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.6696264147758484,
+      "learning_rate": 7.913123252909056e-06,
+      "loss": 0.30917251110076904,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.6794363260269165,
+      "learning_rate": 7.854042905196355e-06,
+      "loss": 0.34172370433807375,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3615648150444031,
+      "learning_rate": 7.794365476788668e-06,
+      "loss": 0.26951897144317627,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4700421988964081,
+      "learning_rate": 7.734103452515976e-06,
+      "loss": 0.33142666816711425,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.452709436416626,
+      "learning_rate": 7.67326943950877e-06,
+      "loss": 0.3427105903625488,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.48398926854133606,
+      "learning_rate": 7.611876164560578e-06,
+      "loss": 0.24839515686035157,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.2996172308921814,
+      "learning_rate": 7.549936471465443e-06,
+      "loss": 0.2821991205215454,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4082246720790863,
+      "learning_rate": 7.487463318330945e-06,
+      "loss": 0.2629000425338745,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6337223052978516,
+      "learning_rate": 7.424469774867291e-06,
+      "loss": 0.25711946487426757,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.7650140523910522,
+      "learning_rate": 7.360969019653068e-06,
+      "loss": 0.4765789031982422,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.6438851356506348,
+      "learning_rate": 7.296974337378209e-06,
+      "loss": 0.27527151107788084,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 1.112627387046814,
+      "learning_rate": 7.2324991160647705e-06,
+      "loss": 0.2819572925567627,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 0.9886193871498108,
+      "learning_rate": 7.1675568442660915e-06,
+      "loss": 0.3160847663879395,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.5070939660072327,
+      "learning_rate": 7.102161108244907e-06,
+      "loss": 0.2885098934173584,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5902522206306458,
+      "learning_rate": 7.036325589131048e-06,
+      "loss": 0.3293052911758423,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5462592244148254,
+      "learning_rate": 6.9700640600592585e-06,
+      "loss": 0.28150961399078367,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.48972848057746887,
+      "learning_rate": 6.903390383287795e-06,
+      "loss": 0.2742903232574463,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6057692766189575,
+      "learning_rate": 6.8363185072983695e-06,
+      "loss": 0.3347393751144409,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.45753175020217896,
+      "learning_rate": 6.7688624638780475e-06,
+      "loss": 0.32706599235534667,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.7748094797134399,
+      "learning_rate": 6.701036365183737e-06,
+      "loss": 0.38280029296875,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.589371383190155,
+      "learning_rate": 6.632854400789835e-06,
+      "loss": 0.3170490264892578,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.5430629849433899,
+      "learning_rate": 6.5643308347197085e-06,
+      "loss": 0.3212059259414673,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.4012330174446106,
+      "learning_rate": 6.495480002461577e-06,
+      "loss": 0.31148293018341067,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5642695426940918,
+      "learning_rate": 6.4263163079694645e-06,
+      "loss": 0.3561388969421387,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.5468930006027222,
+      "learning_rate": 6.356854220649801e-06,
+      "loss": 0.32206296920776367,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.6053391098976135,
+      "learning_rate": 6.28710827233436e-06,
+      "loss": 0.326386547088623,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.7525632977485657,
+      "learning_rate": 6.21709305424012e-06,
+      "loss": 0.3004195213317871,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5053286552429199,
+      "learning_rate": 6.146823213916703e-06,
+      "loss": 0.3025178909301758,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.37086620926856995,
+      "learning_rate": 6.076313452182033e-06,
+      "loss": 0.3551692247390747,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.570372462272644,
+      "learning_rate": 6.005578520046848e-06,
+      "loss": 0.6431361198425293,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.3969172239303589,
+      "learning_rate": 5.934633215628711e-06,
+      "loss": 0.31665098667144775,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5713177919387817,
+      "learning_rate": 5.863492381056164e-06,
+      "loss": 0.29021525382995605,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5128471255302429,
+      "learning_rate": 5.792170899363683e-06,
+      "loss": 0.33125274181365966,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.5780979990959167,
+      "learning_rate": 5.720683691378061e-06,
+      "loss": 0.29233403205871583,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6340335011482239,
+      "learning_rate": 5.6490457125969035e-06,
+      "loss": 0.27084898948669434,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.5844733715057373,
+      "learning_rate": 5.57727195005985e-06,
+      "loss": 0.3200614213943481,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.644882082939148,
+      "learning_rate": 5.505377419213214e-06,
+      "loss": 0.32254467010498045,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.47608786821365356,
+      "learning_rate": 5.433377160768669e-06,
+      "loss": 0.3473681449890137,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.6619153022766113,
+      "learning_rate": 5.361286237556658e-06,
+      "loss": 0.3421504020690918,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.6047444343566895,
+      "learning_rate": 5.289119731375161e-06,
+      "loss": 0.3683443307876587,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 1.0079972743988037,
+      "learning_rate": 5.216892739834519e-06,
+      "loss": 0.3449583053588867,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.5912438035011292,
+      "learning_rate": 5.144620373198916e-06,
+      "loss": 0.38611855506896975,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.6063432693481445,
+      "learning_rate": 5.072317751225254e-06,
+      "loss": 0.37934141159057616,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.4390994906425476,
+      "learning_rate": 5e-06,
+      "loss": 0.27857556343078616,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.6064956784248352,
+      "learning_rate": 4.927682248774747e-06,
+      "loss": 0.3554774284362793,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.6314171552658081,
+      "learning_rate": 4.855379626801085e-06,
+      "loss": 0.30284528732299804,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.671026349067688,
+      "learning_rate": 4.783107260165483e-06,
+      "loss": 0.36607491970062256,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.4459201395511627,
+      "learning_rate": 4.710880268624841e-06,
+      "loss": 0.3821887016296387,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.6088168621063232,
+      "learning_rate": 4.6387137624433446e-06,
+      "loss": 0.3565082550048828,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.5071287751197815,
+      "learning_rate": 4.5666228392313315e-06,
+      "loss": 0.33942298889160155,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7531337141990662,
+      "learning_rate": 4.494622580786788e-06,
+      "loss": 0.3676978588104248,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.5007270574569702,
+      "learning_rate": 4.422728049940151e-06,
+      "loss": 0.28489739894866944,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.42371106147766113,
+      "learning_rate": 4.350954287403099e-06,
+      "loss": 0.2984581232070923,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.563149631023407,
+      "learning_rate": 4.279316308621941e-06,
+      "loss": 0.40024795532226565,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 0.8423017263412476,
+      "learning_rate": 4.207829100636318e-06,
+      "loss": 0.3361294507980347,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5179011821746826,
+      "learning_rate": 4.1365076189438365e-06,
+      "loss": 0.3160973072052002,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.7851564288139343,
+      "learning_rate": 4.06536678437129e-06,
+      "loss": 0.36293606758117675,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5663831233978271,
+      "learning_rate": 3.994421479953153e-06,
+      "loss": 0.3758086681365967,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.6246234178543091,
+      "learning_rate": 3.9236865478179685e-06,
+      "loss": 0.29956531524658203,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5657338500022888,
+      "learning_rate": 3.853176786083299e-06,
+      "loss": 0.37062208652496337,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.780840277671814,
+      "learning_rate": 3.782906945759883e-06,
+      "loss": 0.38033413887023926,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.46465229988098145,
+      "learning_rate": 3.7128917276656406e-06,
+      "loss": 0.2972914218902588,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.6549701690673828,
+      "learning_rate": 3.6431457793502e-06,
+      "loss": 0.38214161396026614,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.7952358722686768,
+      "learning_rate": 3.5736836920305367e-06,
+      "loss": 0.43451919555664065,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.3544764220714569,
+      "learning_rate": 3.5045199975384225e-06,
+      "loss": 0.28854324817657473,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5664554834365845,
+      "learning_rate": 3.4356691652802936e-06,
+      "loss": 0.36930935382843016,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.46652841567993164,
+      "learning_rate": 3.3671455992101665e-06,
+      "loss": 0.3794795036315918,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.5960412621498108,
+      "learning_rate": 3.2989636348162633e-06,
+      "loss": 0.6944560050964356,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5001474022865295,
+      "learning_rate": 3.2311375361219538e-06,
+      "loss": 0.35518996715545653,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.36455702781677246,
+      "learning_rate": 3.163681492701632e-06,
+      "loss": 0.3554563522338867,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.578860342502594,
+      "learning_rate": 3.096609616712207e-06,
+      "loss": 0.4234604835510254,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6481260657310486,
+      "learning_rate": 3.0299359399407436e-06,
+      "loss": 0.34542040824890136,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 1.1381938457489014,
+      "learning_rate": 2.963674410868953e-06,
+      "loss": 0.5298830032348633,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.6089155077934265,
+      "learning_rate": 2.8978388917550936e-06,
+      "loss": 0.33658499717712403,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.7562251687049866,
+      "learning_rate": 2.8324431557339106e-06,
+      "loss": 0.3880928516387939,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.7808563113212585,
+      "learning_rate": 2.767500883935231e-06,
+      "loss": 0.3246309280395508,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.8138899803161621,
+      "learning_rate": 2.7030256626217932e-06,
+      "loss": 0.4132601261138916,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.9273833632469177,
+      "learning_rate": 2.6390309803469326e-06,
+      "loss": 0.4071033477783203,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.5960099101066589,
+      "learning_rate": 2.5755302251327097e-06,
+      "loss": 0.4031569480895996,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.6419629454612732,
+      "learning_rate": 2.5125366816690557e-06,
+      "loss": 0.3036700487136841,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.6673698425292969,
+      "learning_rate": 2.450063528534558e-06,
+      "loss": 0.311791729927063,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.6572533249855042,
+      "learning_rate": 2.3881238354394236e-06,
+      "loss": 0.38104662895202634,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.46291619539260864,
+      "learning_rate": 2.32673056049123e-06,
+      "loss": 0.4063735008239746,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 1.104805827140808,
+      "learning_rate": 2.265896547484026e-06,
+      "loss": 0.36424479484558103,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.6098979115486145,
+      "learning_rate": 2.2056345232113352e-06,
+      "loss": 0.350197958946228,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5803428292274475,
+      "learning_rate": 2.1459570948036486e-06,
+      "loss": 0.364992094039917,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.7771470546722412,
+      "learning_rate": 2.086876747090945e-06,
+      "loss": 0.38045494556427,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.43313416838645935,
+      "learning_rate": 2.0284058399908045e-06,
+      "loss": 0.3114879369735718,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.5160917639732361,
+      "learning_rate": 1.970556605922645e-06,
+      "loss": 0.41294088363647463,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.089125633239746,
+      "learning_rate": 1.9133411472486324e-06,
+      "loss": 0.45021257400512693,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.6227880716323853,
+      "learning_rate": 1.8567714337418092e-06,
+      "loss": 0.3855113506317139,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.8897408246994019,
+      "learning_rate": 1.8008593000819519e-06,
+      "loss": 0.42201600074768064,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.4485880434513092,
+      "learning_rate": 1.7456164433796991e-06,
+      "loss": 0.39100966453552244,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.5533657670021057,
+      "learning_rate": 1.6910544207294521e-06,
+      "loss": 0.33871314525604246,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6669449806213379,
+      "learning_rate": 1.6371846467915603e-06,
+      "loss": 0.35631103515625,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.6646720170974731,
+      "learning_rate": 1.5840183914043338e-06,
+      "loss": 0.3687641143798828,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.9954160451889038,
+      "learning_rate": 1.5315667772263227e-06,
+      "loss": 0.4456006050109863,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.6651373505592346,
+      "learning_rate": 1.4798407774093954e-06,
+      "loss": 0.3330822467803955,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.8815678954124451,
+      "learning_rate": 1.4288512133031073e-06,
+      "loss": 0.4016695022583008,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.6688061952590942,
+      "learning_rate": 1.3786087521908103e-06,
+      "loss": 0.3567208290100098,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 2.1051671504974365,
+      "learning_rate": 1.3291239050580085e-06,
+      "loss": 0.6473428726196289,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.9308188557624817,
+      "learning_rate": 1.2804070243933997e-06,
+      "loss": 0.3749030590057373,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.7065919637680054,
+      "learning_rate": 1.2324683020230798e-06,
+      "loss": 0.733770227432251,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.7088451981544495,
+      "learning_rate": 1.1853177669783645e-06,
+      "loss": 0.37824020385742185,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.7551929950714111,
+      "learning_rate": 1.1389652833976578e-06,
+      "loss": 0.4163835048675537,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.5236889719963074,
+      "learning_rate": 1.0934205484628285e-06,
+      "loss": 0.39182589054107664,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.5291634202003479,
+      "learning_rate": 1.0486930903705095e-06,
+      "loss": 0.3792252540588379,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6111361384391785,
+      "learning_rate": 1.004792266338735e-06,
+      "loss": 0.4349212169647217,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.6383189558982849,
+      "learning_rate": 9.617272606493844e-07,
+      "loss": 0.3830833435058594,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.6607468128204346,
+      "learning_rate": 9.195070827267633e-07,
+      "loss": 0.4139139175415039,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.9052505493164062,
+      "learning_rate": 8.781405652527975e-07,
+      "loss": 0.38831195831298826,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 1.0455974340438843,
+      "learning_rate": 8.376363623191791e-07,
+      "loss": 0.4037032127380371,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.6647090911865234,
+      "learning_rate": 7.980029476168943e-07,
+      "loss": 0.4418759346008301,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.7218889594078064,
+      "learning_rate": 7.59248612663479e-07,
+      "loss": 0.3176127910614014,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 1.124987006187439,
+      "learning_rate": 7.213814650683887e-07,
+      "loss": 0.484192419052124,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.5644972920417786,
+      "learning_rate": 6.844094268368484e-07,
+      "loss": 0.38383920192718507,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.8091310858726501,
+      "learning_rate": 6.483402327125232e-07,
+      "loss": 0.45018877983093264,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.4238279461860657,
+      "learning_rate": 6.131814285593707e-07,
+      "loss": 0.37433390617370604,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.5011367797851562,
+      "learning_rate": 5.789403697830104e-07,
+      "loss": 0.43130178451538087,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.9677279591560364,
+      "learning_rate": 5.456242197919292e-07,
+      "loss": 0.43402600288391113,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.6384739875793457,
+      "learning_rate": 5.132399484988615e-07,
+      "loss": 0.41501555442810056,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.43836286664009094,
+      "learning_rate": 4.817943308626488e-07,
+      "loss": 0.3939836025238037,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.7905747890472412,
+      "learning_rate": 4.5129394547087756e-07,
+      "loss": 0.4381608009338379,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.8479343056678772,
+      "learning_rate": 4.2174517316361075e-07,
+      "loss": 0.42328357696533203,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4482978284358978,
+      "learning_rate": 3.93154195698478e-07,
+      "loss": 0.43547234535217283,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.4706050455570221,
+      "learning_rate": 3.6552699445742026e-07,
+      "loss": 0.4107391834259033,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.6657713055610657,
+      "learning_rate": 3.3886934919535765e-07,
+      "loss": 0.4677100658416748,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.6160376071929932,
+      "learning_rate": 3.131868368310276e-07,
+      "loss": 0.3558232307434082,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.8466643691062927,
+      "learning_rate": 2.8848483028026896e-07,
+      "loss": 0.46170730590820314,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.8067461848258972,
+      "learning_rate": 2.647684973319814e-07,
+      "loss": 0.3582716703414917,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.5093105435371399,
+      "learning_rate": 2.4204279956698994e-07,
+      "loss": 0.3920139312744141,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.7828852534294128,
+      "learning_rate": 2.2031249132006548e-07,
+      "loss": 0.4337261199951172,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.9911346435546875,
+      "learning_rate": 1.9958211868528478e-07,
+      "loss": 0.38247008323669435,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.8960981369018555,
+      "learning_rate": 1.7985601856496947e-07,
+      "loss": 0.3957221508026123,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.7901101112365723,
+      "learning_rate": 1.6113831776238099e-07,
+      "loss": 0.39339218139648435,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.6490103602409363,
+      "learning_rate": 1.4343293211837196e-07,
+      "loss": 0.3751667022705078,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.7808877229690552,
+      "learning_rate": 1.2674356569217282e-07,
+      "loss": 0.355701208114624,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.8131973147392273,
+      "learning_rate": 1.110737099864806e-07,
+      "loss": 0.4032705783843994,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 1.1023856401443481,
+      "learning_rate": 9.642664321702188e-08,
+      "loss": 0.4411326885223389,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.9611843228340149,
+      "learning_rate": 8.280542962673166e-08,
+      "loss": 0.4177103996276855,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.9607189893722534,
+      "learning_rate": 7.02129188446965e-08,
+      "loss": 0.3733290910720825,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 1.1137757301330566,
+      "learning_rate": 5.8651745289997065e-08,
+      "loss": 0.4329118251800537,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.7032621502876282,
+      "learning_rate": 4.8124327620576726e-08,
+      "loss": 0.345641565322876,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.6256678700447083,
+      "learning_rate": 3.863286822724244e-08,
+      "loss": 0.409209156036377,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.6722251772880554,
+      "learning_rate": 3.017935277291495e-08,
+      "loss": 0.3628429889678955,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.5789678692817688,
+      "learning_rate": 2.2765549777217787e-08,
+      "loss": 0.34007580280303956,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 1.0749714374542236,
+      "learning_rate": 1.639301024649209e-08,
+      "loss": 0.4264070510864258,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.5414615273475647,
+      "learning_rate": 1.1063067349318946e-08,
+      "loss": 0.34705543518066406,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.8058632612228394,
+      "learning_rate": 6.7768361376152616e-09,
+      "loss": 0.35213332176208495,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.5058417916297913,
+      "learning_rate": 3.535213313355934e-09,
+      "loss": 0.39220623970031737,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 1.0682101249694824,
+      "learning_rate": 1.3388770409805685e-09,
+      "loss": 0.45152854919433594,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.6894797086715698,
+      "learning_rate": 1.8828680551918887e-10,
+      "loss": 0.3403477668762207,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 400,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532391257768261e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/training_args.bin b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..649ae980fcab789df31d5926f8d4ba0e31d75659
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/checkpoint-1207/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71371688dfd537b126d4621e611796dedc057539dcf7397af2bc1955bebc8155
+size 5265
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6febaf9670885d9e88916b73da4a5bbf938e5bfb
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e95c6fcdb83e6228d0fd8c74313697d9f44fd141
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.1"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/model.safetensors b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..632793bcde23c5c657dff0223d952f21886a36d0
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d0ec7951180204a7e052f891ee3f23271f5a89b910494bec96c02527b0cc02c
+size 1867938664
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ac718f112af542c39adcd73bb3bde829991ce0c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf03774334462d6e34b5005cb11120a62275f146ee2953e68731ecdbce84fbb
+size 11423222
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train.log b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train.log
new file mode 100644
index 0000000000000000000000000000000000000000..73039a50e76bd8d41c0b397ba00b765bc21c919a
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train.log
@@ -0,0 +1,511 @@
+[transformers] warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
+[transformers] `MossSpeakerRopeProcessor` defines `feature_extractor_class = 'AutoFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
+[transformers] You are using a model of type `moss_speaker_rope` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.
+out_channels : 224
+
Loading weights:   0%|          | 0/1518 [00:00<?, ?it/s]
Loading weights:   3%|▎         | 47/1518 [00:00<00:04, 334.67it/s]
Loading weights:   5%|▌         | 81/1518 [00:00<00:04, 296.29it/s]
Loading weights:   8%|▊         | 114/1518 [00:00<00:05, 273.98it/s]
Loading weights:  11%|█         | 164/1518 [00:00<00:04, 283.45it/s]
Loading weights:  13%|█▎        | 193/1518 [00:00<00:08, 163.26it/s]
Loading weights:  14%|█▍        | 214/1518 [00:01<00:09, 140.89it/s]
Loading weights:  15%|█▌        | 232/1518 [00:01<00:11, 115.57it/s]
Loading weights:  17%|█▋        | 251/1518 [00:01<00:10, 122.13it/s]
Loading weights:  18%|█▊        | 266/1518 [00:01<00:11, 105.16it/s]
Loading weights:  18%|█▊        | 278/1518 [00:01<00:11, 104.22it/s]
Loading weights:  19%|█▉        | 290/1518 [00:02<00:12, 97.94it/s] 
Loading weights:  20%|██        | 304/1518 [00:02<00:11, 106.55it/s]
Loading weights:  25%|██▍       | 377/1518 [00:02<00:04, 245.81it/s]
Loading weights:  43%|████▎     | 657/1518 [00:02<00:01, 845.28it/s]
Loading weights:  61%|██████    | 929/1518 [00:02<00:00, 1318.07it/s]
Loading weights:  71%|███████   | 1079/1518 [00:02<00:00, 1210.29it/s]
Loading weights:  80%|███████▉  | 1214/1518 [00:03<00:00, 408.03it/s] 
Loading weights:  86%|████████▋ | 1313/1518 [00:04<00:00, 262.74it/s]
Loading weights:  91%|█████████▏| 1386/1518 [00:05<00:00, 205.23it/s]
Loading weights:  95%|█████████▍| 1440/1518 [00:05<00:00, 171.16it/s]
Loading weights:  98%|█████████▊| 1481/1518 [00:06<00:00, 143.82it/s]
Loading weights: 100%|█████████▉| 1512/1518 [00:06<00:00, 149.29it/s]
Loading weights: 100%|██████████| 1518/1518 [00:06<00:00, 240.29it/s]
+
  0%|          | 0/1207 [00:00<?, ?it/s][transformers] `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.
+/wangshuai/moss/MOSS_Speaker-RoPE/moss_speaker_rope/wespeaker_redimnet/tfmel.py:296: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+
  0%|          | 1/1207 [01:05<21:48:46, 65.11s/it]
  0%|          | 2/1207 [01:40<15:58:08, 47.71s/it]
  0%|          | 3/1207 [02:14<13:46:29, 41.19s/it]
  0%|          | 4/1207 [02:47<12:44:10, 38.11s/it]
  0%|          | 5/1207 [03:34<13:48:08, 41.34s/it]step=    5  loss=  0.4680  grad_norm= 0.909  lr=3.31e-07  spk_acc=3607/3917
+
                                                   

  0%|          | 5/1207 [03:34<13:48:08, 41.34s/it]
  0%|          | 6/1207 [04:08<12:56:44, 38.80s/it]
  1%|          | 7/1207 [04:35<11:42:24, 35.12s/it]
  1%|          | 8/1207 [05:05<11:04:22, 33.25s/it]
  1%|          | 9/1207 [05:38<11:03:26, 33.23s/it]
  1%|          | 10/1207 [06:17<11:37:54, 34.98s/it]{'loss': '0.468', 'grad_norm': '0.9088', 'learning_rate': '3.306e-07', 'epoch': '0.004143'}
+step=   10  loss=  0.3254  grad_norm= 0.419  lr=7.44e-07  spk_acc=2181/2271
+
                                                    

  1%|          | 10/1207 [06:17<11:37:54, 34.98s/it]
  1%|          | 11/1207 [06:50<11:29:36, 34.60s/it]
  1%|          | 12/1207 [07:29<11:54:48, 35.89s/it]
  1%|          | 13/1207 [08:11<12:27:18, 37.55s/it]
  1%|          | 14/1207 [08:44<11:58:44, 36.15s/it]
  1%|          | 15/1207 [09:21<12:07:28, 36.62s/it]{'loss': '0.3254', 'grad_norm': '0.4195', 'learning_rate': '7.438e-07', 'epoch': '0.008285'}
+step=   15  loss=  0.4408  grad_norm= 0.486  lr=1.16e-06  spk_acc=2399/2524
+
                                                    

  1%|          | 15/1207 [09:21<12:07:28, 36.62s/it]
  1%|▏         | 16/1207 [10:00<12:18:17, 37.19s/it]
  1%|▏         | 17/1207 [10:38<12:25:31, 37.59s/it]
  1%|▏         | 18/1207 [11:12<12:01:58, 36.43s/it]
  2%|▏         | 19/1207 [11:49<12:01:07, 36.42s/it]
  2%|▏         | 20/1207 [12:13<10:47:22, 32.72s/it]{'loss': '0.4408', 'grad_norm': '0.486', 'learning_rate': '1.157e-06', 'epoch': '0.01243'}
+step=   20  loss=  0.3342  grad_norm= 0.615  lr=1.57e-06  spk_acc=2137/2223
+
                                                    

  2%|▏         | 20/1207 [12:13<10:47:22, 32.72s/it]
  2%|▏         | 21/1207 [12:50<11:15:25, 34.17s/it]
  2%|▏         | 22/1207 [13:28<11:36:14, 35.25s/it]
  2%|▏         | 23/1207 [14:12<12:30:06, 38.01s/it]
  2%|▏         | 24/1207 [14:47<12:07:41, 36.91s/it]
  2%|▏         | 25/1207 [15:23<12:05:50, 36.84s/it]{'loss': '0.3342', 'grad_norm': '0.6155', 'learning_rate': '1.57e-06', 'epoch': '0.01657'}
+step=   25  loss=  0.4068  grad_norm= 0.443  lr=1.98e-06  spk_acc=3041/3216
+
                                                    

  2%|▏         | 25/1207 [15:23<12:05:50, 36.84s/it]
  2%|▏         | 26/1207 [15:59<11:56:16, 36.39s/it]
  2%|▏         | 27/1207 [16:32<11:37:01, 35.44s/it]
  2%|▏         | 28/1207 [17:06<11:27:36, 34.99s/it]
  2%|▏         | 29/1207 [17:44<11:44:05, 35.86s/it]
  2%|▏         | 30/1207 [18:26<12:23:25, 37.90s/it]{'loss': '0.4068', 'grad_norm': '0.4429', 'learning_rate': '1.983e-06', 'epoch': '0.02071'}
+step=   30  loss=  0.4108  grad_norm= 0.778  lr=2.40e-06  spk_acc=2830/2995
+
                                                    

  2%|▏         | 30/1207 [18:26<12:23:25, 37.90s/it]
  3%|▎         | 31/1207 [19:07<12:39:48, 38.77s/it]
  3%|▎         | 32/1207 [19:40<12:03:06, 36.92s/it]
  3%|▎         | 33/1207 [20:17<12:03:05, 36.96s/it]
  3%|▎         | 34/1207 [20:48<11:25:58, 35.09s/it]
  3%|▎         | 35/1207 [21:26<11:43:48, 36.03s/it]{'loss': '0.4108', 'grad_norm': '0.778', 'learning_rate': '2.397e-06', 'epoch': '0.02486'}
+step=   35  loss=  0.4021  grad_norm= 0.423  lr=2.81e-06  spk_acc=3292/3525
+
                                                    

  3%|▎         | 35/1207 [21:26<11:43:48, 36.03s/it]
  3%|▎         | 36/1207 [22:05<12:01:00, 36.94s/it]
  3%|▎         | 37/1207 [22:37<11:32:18, 35.50s/it]
  3%|▎         | 38/1207 [23:13<11:33:18, 35.58s/it]
  3%|▎         | 39/1207 [23:49<11:38:10, 35.87s/it]
  3%|▎         | 40/1207 [24:20<11:09:55, 34.44s/it]{'loss': '0.4021', 'grad_norm': '0.4232', 'learning_rate': '2.81e-06', 'epoch': '0.029'}
+step=   40  loss=  0.4045  grad_norm= 0.533  lr=3.22e-06  spk_acc=2111/2246
+
                                                    

  3%|▎         | 40/1207 [24:20<11:09:55, 34.44s/it]
  3%|▎         | 41/1207 [24:47<10:23:53, 32.10s/it]
  3%|▎         | 42/1207 [25:14<9:54:59, 30.64s/it] 
  4%|▎         | 43/1207 [25:47<10:06:30, 31.26s/it]
  4%|▎         | 44/1207 [26:19<10:10:51, 31.51s/it]
  4%|▎         | 45/1207 [26:56<10:43:10, 33.21s/it]{'loss': '0.4045', 'grad_norm': '0.533', 'learning_rate': '3.223e-06', 'epoch': '0.03314'}
+step=   45  loss=  0.4133  grad_norm= 0.409  lr=3.64e-06  spk_acc=1898/2040
+
                                                    

  4%|▎         | 45/1207 [26:56<10:43:10, 33.21s/it]
  4%|▍         | 46/1207 [27:30<10:42:15, 33.19s/it]
  4%|▍         | 47/1207 [28:03<10:45:38, 33.40s/it]
  4%|▍         | 48/1207 [28:38<10:53:59, 33.86s/it]
  4%|▍         | 49/1207 [29:12<10:49:57, 33.68s/it]
  4%|▍         | 50/1207 [29:44<10:40:32, 33.22s/it]{'loss': '0.4133', 'grad_norm': '0.409', 'learning_rate': '3.636e-06', 'epoch': '0.03728'}
+step=   50  loss=  0.3559  grad_norm= 0.703  lr=4.05e-06  spk_acc=2144/2205
+
                                                    

  4%|▍         | 50/1207 [29:44<10:40:32, 33.22s/it]
  4%|▍         | 51/1207 [30:18<10:44:19, 33.44s/it]
  4%|▍         | 52/1207 [30:50<10:35:01, 32.99s/it]
  4%|▍         | 53/1207 [31:22<10:33:11, 32.92s/it]
  4%|▍         | 54/1207 [32:02<11:12:51, 35.01s/it]
  5%|▍         | 55/1207 [32:39<11:19:30, 35.39s/it]{'loss': '0.3559', 'grad_norm': '0.7031', 'learning_rate': '4.05e-06', 'epoch': '0.04143'}
+step=   55  loss=  0.3346  grad_norm= 0.537  lr=4.46e-06  spk_acc=2153/2237
+
                                                    

  5%|▍         | 55/1207 [32:39<11:19:30, 35.39s/it]
  5%|▍         | 56/1207 [33:11<11:01:02, 34.46s/it]
  5%|▍         | 57/1207 [33:45<10:57:35, 34.31s/it]
  5%|▍         | 58/1207 [34:20<11:00:07, 34.47s/it]
  5%|▍         | 59/1207 [35:03<11:48:24, 37.02s/it]
  5%|▍         | 60/1207 [35:40<11:51:50, 37.24s/it]{'loss': '0.3346', 'grad_norm': '0.5372', 'learning_rate': '4.463e-06', 'epoch': '0.04557'}
+step=   60  loss=  0.3522  grad_norm= 0.452  lr=4.88e-06  spk_acc=2561/2663
+
                                                    

  5%|▍         | 60/1207 [35:40<11:51:50, 37.24s/it]
  5%|▌         | 61/1207 [36:18<11:52:55, 37.33s/it]
  5%|▌         | 62/1207 [36:49<11:15:54, 35.42s/it]
  5%|▌         | 63/1207 [37:24<11:12:51, 35.29s/it]
  5%|▌         | 64/1207 [37:59<11:11:16, 35.24s/it]
  5%|▌         | 65/1207 [38:38<11:30:04, 36.26s/it]{'loss': '0.3522', 'grad_norm': '0.4524', 'learning_rate': '4.876e-06', 'epoch': '0.04971'}
+step=   65  loss=  0.3251  grad_norm= 0.406  lr=5.29e-06  spk_acc=2247/2335
+
                                                    

  5%|▌         | 65/1207 [38:38<11:30:04, 36.26s/it]
  5%|▌         | 66/1207 [39:11<11:14:10, 35.45s/it]
  6%|▌         | 67/1207 [39:48<11:20:03, 35.79s/it]
  6%|▌         | 68/1207 [40:20<10:57:41, 34.65s/it]
  6%|▌         | 69/1207 [40:51<10:39:45, 33.73s/it]
  6%|▌         | 70/1207 [41:27<10:47:42, 34.18s/it]{'loss': '0.3251', 'grad_norm': '0.4058', 'learning_rate': '5.289e-06', 'epoch': '0.05385'}
+step=   70  loss=  0.2527  grad_norm= 0.650  lr=5.70e-06  spk_acc=1622/1635
+
                                                    

  6%|▌         | 70/1207 [41:27<10:47:42, 34.18s/it]
  6%|▌         | 71/1207 [42:02<10:53:12, 34.50s/it]
  6%|▌         | 72/1207 [42:37<10:55:31, 34.65s/it]
  6%|▌         | 73/1207 [43:08<10:34:04, 33.55s/it]
  6%|▌         | 74/1207 [43:41<10:30:46, 33.40s/it]
  6%|▌         | 75/1207 [44:13<10:24:33, 33.10s/it]{'loss': '0.2527', 'grad_norm': '0.6495', 'learning_rate': '5.702e-06', 'epoch': '0.058'}
+step=   75  loss=  0.2897  grad_norm= 0.566  lr=6.12e-06  spk_acc=2035/2078
+
                                                    

  6%|▌         | 75/1207 [44:13<10:24:33, 33.10s/it]
  6%|▋         | 76/1207 [44:50<10:42:25, 34.08s/it]
  6%|▋         | 77/1207 [45:27<10:57:57, 34.94s/it]
  6%|▋         | 78/1207 [46:04<11:12:06, 35.72s/it]
  7%|▋         | 79/1207 [46:32<10:27:47, 33.39s/it]
  7%|▋         | 80/1207 [46:59<9:51:13, 31.48s/it] {'loss': '0.2897', 'grad_norm': '0.5655', 'learning_rate': '6.116e-06', 'epoch': '0.06214'}
+step=   80  loss=  0.3527  grad_norm= 0.985  lr=6.53e-06  spk_acc=2663/2793
+
                                                   

  7%|▋         | 80/1207 [46:59<9:51:13, 31.48s/it]
  7%|▋         | 81/1207 [47:32<10:00:31, 32.00s/it]
  7%|▋         | 82/1207 [48:10<10:31:47, 33.70s/it]
  7%|▋         | 83/1207 [48:40<10:12:05, 32.67s/it]
  7%|▋         | 84/1207 [49:14<10:18:31, 33.05s/it]
  7%|▋         | 85/1207 [49:48<10:20:35, 33.19s/it]{'loss': '0.3527', 'grad_norm': '0.9851', 'learning_rate': '6.529e-06', 'epoch': '0.06628'}
+step=   85  loss=  0.3311  grad_norm= 0.591  lr=6.94e-06  spk_acc=2450/2627
+
                                                    

  7%|▋         | 85/1207 [49:48<10:20:35, 33.19s/it]
  7%|▋         | 86/1207 [50:18<10:02:21, 32.24s/it]
  7%|▋         | 87/1207 [50:50<10:01:42, 32.23s/it]
  7%|▋         | 88/1207 [51:24<10:12:06, 32.82s/it]
  7%|▋         | 89/1207 [51:32<7:53:58, 25.44s/it] 
  7%|▋         | 90/1207 [52:04<8:30:05, 27.40s/it]{'loss': '0.3311', 'grad_norm': '0.5908', 'learning_rate': '6.942e-06', 'epoch': '0.07042'}
+step=   90  loss=  0.6235  grad_norm= 0.346  lr=7.36e-06  spk_acc=1969/2062
+
                                                   

  7%|▋         | 90/1207 [52:04<8:30:05, 27.40s/it]
  8%|▊         | 91/1207 [52:45<9:45:09, 31.46s/it]
  8%|▊         | 92/1207 [53:19<9:57:34, 32.16s/it]
  8%|▊         | 93/1207 [53:53<10:09:55, 32.85s/it]
  8%|▊         | 94/1207 [54:27<10:15:32, 33.18s/it]
  8%|▊         | 95/1207 [55:13<11:22:09, 36.81s/it]{'loss': '0.6235', 'grad_norm': '0.3464', 'learning_rate': '7.355e-06', 'epoch': '0.07457'}
+step=   95  loss=  0.3618  grad_norm= 0.651  lr=7.77e-06  spk_acc=3306/3480
+
                                                    

  8%|▊         | 95/1207 [55:13<11:22:09, 36.81s/it]
  8%|▊         | 96/1207 [56:03<12:37:02, 40.88s/it]
  8%|▊         | 97/1207 [56:45<12:43:28, 41.27s/it]
  8%|▊         | 98/1207 [57:25<12:34:21, 40.81s/it]
  8%|▊         | 99/1207 [58:03<12:19:11, 40.03s/it]
  8%|▊         | 100/1207 [58:40<12:00:21, 39.04s/it]{'loss': '0.3618', 'grad_norm': '0.6509', 'learning_rate': '7.769e-06', 'epoch': '0.07871'}
+step=  100  loss=  0.3799  grad_norm= 0.417  lr=8.18e-06  spk_acc=4018/4175
+
                                                     

  8%|▊         | 100/1207 [58:40<12:00:21, 39.04s/it]
  8%|▊         | 101/1207 [59:13<11:25:57, 37.21s/it]
  8%|▊         | 102/1207 [59:47<11:08:48, 36.32s/it]
  9%|▊         | 103/1207 [1:00:17<10:31:21, 34.31s/it]
  9%|▊         | 104/1207 [1:00:57<11:01:52, 36.00s/it]
  9%|▊         | 105/1207 [1:01:29<10:41:09, 34.91s/it]{'loss': '0.3799', 'grad_norm': '0.4168', 'learning_rate': '8.182e-06', 'epoch': '0.08285'}
+step=  105  loss=  0.3221  grad_norm= 0.495  lr=8.60e-06  spk_acc=2477/2605
+
                                                       

  9%|▊         | 105/1207 [1:01:29<10:41:09, 34.91s/it]
  9%|▉         | 106/1207 [1:02:02<10:27:33, 34.20s/it]
  9%|▉         | 107/1207 [1:02:37<10:32:33, 34.50s/it]
  9%|▉         | 108/1207 [1:03:10<10:27:03, 34.23s/it]
  9%|▉         | 109/1207 [1:03:47<10:41:16, 35.04s/it]
  9%|▉         | 110/1207 [1:04:14<9:56:52, 32.65s/it] {'loss': '0.3221', 'grad_norm': '0.4955', 'learning_rate': '8.595e-06', 'epoch': '0.08699'}
+step=  110  loss=  0.3425  grad_norm= 0.388  lr=9.01e-06  spk_acc=2698/2828
+
                                                      

  9%|▉         | 110/1207 [1:04:14<9:56:52, 32.65s/it]
  9%|▉         | 111/1207 [1:04:45<9:45:52, 32.07s/it]
  9%|▉         | 112/1207 [1:05:19<9:56:38, 32.69s/it]
  9%|▉         | 113/1207 [1:05:52<9:58:53, 32.85s/it]
  9%|▉         | 114/1207 [1:06:26<10:02:53, 33.10s/it]
 10%|▉         | 115/1207 [1:06:57<9:51:55, 32.52s/it] {'loss': '0.3425', 'grad_norm': '0.3881', 'learning_rate': '9.008e-06', 'epoch': '0.09114'}
+step=  115  loss=  0.3067  grad_norm= 0.420  lr=9.42e-06  spk_acc=2665/2774
+
                                                      

 10%|▉         | 115/1207 [1:06:57<9:51:55, 32.52s/it]
 10%|▉         | 116/1207 [1:07:30<9:50:47, 32.49s/it]
 10%|▉         | 117/1207 [1:08:08<10:19:31, 34.10s/it]
 10%|▉         | 118/1207 [1:08:36<9:45:59, 32.29s/it] 
 10%|▉         | 119/1207 [1:09:10<9:54:00, 32.76s/it]
 10%|▉         | 120/1207 [1:09:44<10:02:07, 33.24s/it]{'loss': '0.3067', 'grad_norm': '0.4198', 'learning_rate': '9.421e-06', 'epoch': '0.09528'}
+step=  120  loss=  0.2994  grad_norm= 0.432  lr=9.83e-06  spk_acc=2466/2494
+
                                                       

 10%|▉         | 120/1207 [1:09:44<10:02:07, 33.24s/it]
 10%|█         | 121/1207 [1:10:19<10:13:32, 33.90s/it]
 10%|█         | 122/1207 [1:10:58<10:40:42, 35.43s/it]
 10%|█         | 123/1207 [1:11:44<11:33:14, 38.37s/it]
 10%|█         | 124/1207 [1:12:19<11:14:27, 37.37s/it]
 10%|█         | 125/1207 [1:12:56<11:15:28, 37.46s/it]{'loss': '0.2994', 'grad_norm': '0.432', 'learning_rate': '9.835e-06', 'epoch': '0.09942'}
+step=  125  loss=  0.3241  grad_norm= 0.430  lr=1.00e-05  spk_acc=2779/2973
+
                                                       

 10%|█         | 125/1207 [1:12:56<11:15:28, 37.46s/it]
 10%|█         | 126/1207 [1:13:29<10:52:07, 36.20s/it]
 11%|█         | 127/1207 [1:14:06<10:55:19, 36.41s/it]
 11%|█         | 128/1207 [1:14:42<10:52:11, 36.27s/it]
 11%|█         | 129/1207 [1:15:31<11:56:37, 39.89s/it]
 11%|█         | 130/1207 [1:16:05<11:27:26, 38.30s/it]{'loss': '0.3241', 'grad_norm': '0.4299', 'learning_rate': '1e-05', 'epoch': '0.1036'}
+step=  130  loss=  0.3409  grad_norm= 0.538  lr=1.00e-05  spk_acc=3624/3808
+
                                                       

 11%|█         | 130/1207 [1:16:05<11:27:26, 38.30s/it]
 11%|█         | 131/1207 [1:16:37<10:53:49, 36.46s/it]
 11%|█         | 132/1207 [1:17:10<10:33:12, 35.34s/it]
 11%|█         | 133/1207 [1:17:43<10:17:25, 34.49s/it]
 11%|█         | 134/1207 [1:18:14<10:02:10, 33.67s/it]
 11%|█         | 135/1207 [1:19:13<12:12:35, 41.00s/it]{'loss': '0.3409', 'grad_norm': '0.5378', 'learning_rate': '9.999e-06', 'epoch': '0.1077'}
+step=  135  loss=  0.2805  grad_norm= 0.341  lr=1.00e-05  spk_acc=1859/1926
+
                                                       

 11%|█         | 135/1207 [1:19:13<12:12:35, 41.00s/it]
 11%|█▏        | 136/1207 [1:19:55<12:20:02, 41.46s/it]
 11%|█▏        | 137/1207 [1:20:30<11:46:05, 39.59s/it]
 11%|█▏        | 138/1207 [1:21:08<11:37:09, 39.13s/it]
 12%|█▏        | 139/1207 [1:21:42<11:09:08, 37.59s/it]
 12%|█▏        | 140/1207 [1:22:08<10:02:37, 33.89s/it]{'loss': '0.2805', 'grad_norm': '0.3412', 'learning_rate': '9.996e-06', 'epoch': '0.1118'}
+step=  140  loss=  0.3481  grad_norm= 0.443  lr=9.99e-06  spk_acc=2882/3023
+
                                                       

 12%|█▏        | 140/1207 [1:22:08<10:02:37, 33.89s/it]
 12%|█▏        | 141/1207 [1:22:38<9:41:24, 32.72s/it] 
 12%|█▏        | 142/1207 [1:23:15<10:07:51, 34.25s/it]
 12%|█▏        | 143/1207 [1:23:48<9:57:20, 33.68s/it] 
 12%|█▏        | 144/1207 [1:24:21<9:54:38, 33.56s/it]
 12%|█▏        | 145/1207 [1:25:00<10:21:06, 35.09s/it]{'loss': '0.3481', 'grad_norm': '0.4428', 'learning_rate': '9.993e-06', 'epoch': '0.116'}
+step=  145  loss=  0.2797  grad_norm= 0.359  lr=9.99e-06  spk_acc=2395/2424
+
                                                       

 12%|█▏        | 145/1207 [1:25:00<10:21:06, 35.09s/it]
 12%|█▏        | 146/1207 [1:25:33<10:12:28, 34.64s/it]
 12%|█▏        | 147/1207 [1:26:09<10:15:57, 34.87s/it]
 12%|█▏        | 148/1207 [1:26:43<10:13:13, 34.74s/it]
 12%|█▏        | 149/1207 [1:27:23<10:37:08, 36.13s/it]
 12%|█▏        | 150/1207 [1:27:55<10:14:56, 34.91s/it]{'loss': '0.2797', 'grad_norm': '0.3594', 'learning_rate': '9.989e-06', 'epoch': '0.1201'}
+step=  150  loss=  0.3594  grad_norm= 0.557  lr=9.98e-06  spk_acc=2894/3069
+
                                                       

 12%|█▏        | 150/1207 [1:27:55<10:14:56, 34.91s/it]
 13%|█▎        | 151/1207 [1:28:30<10:15:11, 34.95s/it]
 13%|█▎        | 152/1207 [1:29:00<9:53:00, 33.73s/it] 
 13%|█▎        | 153/1207 [1:29:39<10:19:00, 35.24s/it]
 13%|█▎        | 154/1207 [1:30:13<10:09:01, 34.70s/it]
 13%|█▎        | 155/1207 [1:30:49<10:18:30, 35.28s/it]{'loss': '0.3594', 'grad_norm': '0.5566', 'learning_rate': '9.984e-06', 'epoch': '0.1243'}
+step=  155  loss=  0.3341  grad_norm= 0.496  lr=9.98e-06  spk_acc=2649/2778
+
                                                       

 13%|█▎        | 155/1207 [1:30:49<10:18:30, 35.28s/it]
 13%|█▎        | 156/1207 [1:31:26<10:25:11, 35.69s/it]
 13%|█▎        | 157/1207 [1:32:01<10:18:45, 35.36s/it]
 13%|█▎        | 158/1207 [1:32:36<10:20:15, 35.48s/it]
 13%|█▎        | 159/1207 [1:33:17<10:48:38, 37.14s/it]
 13%|█▎        | 160/1207 [1:33:49<10:18:29, 35.44s/it]{'loss': '0.3341', 'grad_norm': '0.496', 'learning_rate': '9.977e-06', 'epoch': '0.1284'}
+step=  160  loss=  0.3541  grad_norm= 0.546  lr=9.97e-06  spk_acc=2627/2816
+
                                                       

 13%|█▎        | 160/1207 [1:33:49<10:18:29, 35.44s/it]
 13%|█▎        | 161/1207 [1:34:24<10:14:54, 35.27s/it]
 13%|█▎        | 162/1207 [1:34:55<9:51:14, 33.95s/it] 
 14%|█▎        | 163/1207 [1:35:26<9:39:46, 33.32s/it]
 14%|█▎        | 164/1207 [1:36:02<9:48:47, 33.87s/it]
 14%|█▎        | 165/1207 [1:36:37<9:54:23, 34.23s/it]{'loss': '0.3541', 'grad_norm': '0.5456', 'learning_rate': '9.97e-06', 'epoch': '0.1326'}
+step=  165  loss=  0.2924  grad_norm= 0.429  lr=9.96e-06  spk_acc=2153/2206
+
                                                      

 14%|█▎        | 165/1207 [1:36:37<9:54:23, 34.23s/it]
 14%|█▍        | 166/1207 [1:37:09<9:42:19, 33.56s/it]
 14%|█▍        | 167/1207 [1:37:38<9:18:26, 32.22s/it]
 14%|█▍        | 168/1207 [1:38:08<9:06:42, 31.57s/it]
 14%|█▍        | 169/1207 [1:38:45<9:37:00, 33.35s/it]
 14%|█▍        | 170/1207 [1:39:31<10:40:37, 37.07s/it]{'loss': '0.2924', 'grad_norm': '0.4292', 'learning_rate': '9.961e-06', 'epoch': '0.1367'}
+step=  170  loss=  0.3786  grad_norm= 1.151  lr=9.95e-06  spk_acc=2357/2470
+
                                                       

 14%|█▍        | 170/1207 [1:39:31<10:40:37, 37.07s/it]
 14%|█▍        | 171/1207 [1:40:03<10:11:34, 35.42s/it]
 14%|█▍        | 172/1207 [1:40:42<10:30:08, 36.53s/it]
 14%|█▍        | 173/1207 [1:41:20<10:41:03, 37.20s/it]
 14%|█▍        | 174/1207 [1:41:57<10:38:36, 37.09s/it]
 14%|█▍        | 175/1207 [1:42:31<10:21:59, 36.16s/it]{'loss': '0.3786', 'grad_norm': '1.151', 'learning_rate': '9.952e-06', 'epoch': '0.1408'}
+step=  175  loss=  0.3233  grad_norm= 0.384  lr=9.94e-06  spk_acc=3201/3367
+
                                                       

 14%|█▍        | 175/1207 [1:42:31<10:21:59, 36.16s/it]
 15%|█▍        | 176/1207 [1:43:03<9:58:12, 34.81s/it] 
 15%|█▍        | 177/1207 [1:43:39<10:02:22, 35.09s/it]
 15%|█▍        | 178/1207 [1:44:23<10:48:44, 37.83s/it]
 15%|█▍        | 179/1207 [1:44:53<10:10:30, 35.63s/it]
 15%|█▍        | 180/1207 [1:45:27<10:00:52, 35.10s/it]{'loss': '0.3233', 'grad_norm': '0.3837', 'learning_rate': '9.941e-06', 'epoch': '0.145'}
+step=  180  loss=  0.2559  grad_norm= 0.381  lr=9.93e-06  spk_acc=2631/2669
+
                                                       

 15%|█▍        | 180/1207 [1:45:27<10:00:52, 35.10s/it]
 15%|█▍        | 181/1207 [1:45:57<9:34:59, 33.63s/it] 
 15%|█▌        | 182/1207 [1:46:33<9:44:52, 34.24s/it]
 15%|█▌        | 183/1207 [1:47:15<10:21:27, 36.41s/it]
 15%|█▌        | 184/1207 [1:47:48<10:05:24, 35.51s/it]
 15%|█▌        | 185/1207 [1:48:19<9:41:19, 34.13s/it] {'loss': '0.2559', 'grad_norm': '0.381', 'learning_rate': '9.93e-06', 'epoch': '0.1491'}
+step=  185  loss=  0.3104  grad_norm= 0.422  lr=9.92e-06  spk_acc=2841/2974
+
                                                      

 15%|█▌        | 185/1207 [1:48:19<9:41:19, 34.13s/it]
 15%|█▌        | 186/1207 [1:48:50<9:23:12, 33.10s/it]
 15%|█▌        | 187/1207 [1:49:27<9:46:18, 34.49s/it]
 16%|█▌        | 188/1207 [1:49:58<9:25:07, 33.28s/it]
 16%|█▌        | 189/1207 [1:50:31<9:26:16, 33.38s/it]
 16%|█▌        | 190/1207 [1:51:06<9:29:29, 33.60s/it]{'loss': '0.3104', 'grad_norm': '0.4218', 'learning_rate': '9.917e-06', 'epoch': '0.1533'}
+step=  190  loss=  0.2948  grad_norm= 0.435  lr=9.90e-06  spk_acc=3023/3166
+
                                                      

 16%|█▌        | 190/1207 [1:51:06<9:29:29, 33.60s/it]
 16%|█▌        | 191/1207 [1:51:36<9:11:32, 32.57s/it]
 16%|█▌        | 192/1207 [1:52:05<8:52:26, 31.47s/it]
 16%|█▌        | 193/1207 [1:52:41<9:14:15, 32.80s/it]
 16%|█▌        | 194/1207 [1:53:19<9:43:23, 34.55s/it]
 16%|█▌        | 195/1207 [1:53:55<9:50:58, 35.04s/it]{'loss': '0.2948', 'grad_norm': '0.4353', 'learning_rate': '9.904e-06', 'epoch': '0.1574'}
+step=  195  loss=  0.3046  grad_norm= 0.771  lr=9.89e-06  spk_acc=2214/2298
+
                                                      

 16%|█▌        | 195/1207 [1:53:55<9:50:58, 35.04s/it]
 16%|█▌        | 196/1207 [1:54:33<10:05:18, 35.92s/it]
 16%|█▋        | 197/1207 [1:55:07<9:54:56, 35.34s/it] 
 16%|█▋        | 198/1207 [1:55:41<9:45:37, 34.82s/it]
 16%|█▋        | 199/1207 [1:56:22<10:18:00, 36.79s/it]
 17%|█▋        | 200/1207 [1:56:59<10:15:39, 36.68s/it]{'loss': '0.3046', 'grad_norm': '0.7708', 'learning_rate': '9.889e-06', 'epoch': '0.1616'}
+step=  200  loss=  0.2732  grad_norm= 0.348  lr=9.87e-06  spk_acc=2968/3021
+
                                                       

 17%|█▋        | 200/1207 [1:56:59<10:15:39, 36.68s/it]
 17%|█▋        | 201/1207 [1:57:28<9:35:56, 34.35s/it] 
 17%|█▋        | 202/1207 [1:58:01<9:28:04, 33.92s/it]
 17%|█▋        | 203/1207 [1:58:35<9:30:44, 34.11s/it]
 17%|█▋        | 204/1207 [1:59:11<9:38:14, 34.59s/it]
 17%|█▋        | 205/1207 [1:59:49<9:53:37, 35.55s/it]{'loss': '0.2732', 'grad_norm': '0.3478', 'learning_rate': '9.873e-06', 'epoch': '0.1657'}
+step=  205  loss=  0.2296  grad_norm= 0.342  lr=9.86e-06  spk_acc=2417/2442
+
                                                      

 17%|█▋        | 205/1207 [1:59:49<9:53:37, 35.55s/it]
 17%|█▋        | 206/1207 [2:00:21<9:36:28, 34.55s/it]
 17%|█▋        | 207/1207 [2:00:48<8:59:40, 32.38s/it]
 17%|█▋        | 208/1207 [2:01:20<8:58:13, 32.33s/it]
 17%|█▋        | 209/1207 [2:01:59<9:30:42, 34.31s/it]
 17%|█▋        | 210/1207 [2:02:33<9:25:55, 34.06s/it]{'loss': '0.2296', 'grad_norm': '0.3421', 'learning_rate': '9.857e-06', 'epoch': '0.1698'}
+step=  210  loss=  0.2872  grad_norm= 0.435  lr=9.84e-06  spk_acc=2075/2180
+
                                                      

 17%|█▋        | 210/1207 [2:02:33<9:25:55, 34.06s/it]
 17%|█▋        | 211/1207 [2:03:06<9:20:49, 33.78s/it]
 18%|█▊        | 212/1207 [2:03:39<9:15:12, 33.48s/it]
 18%|█▊        | 213/1207 [2:04:11<9:07:49, 33.07s/it]
 18%|█▊        | 214/1207 [2:04:39<8:44:26, 31.69s/it]
 18%|█▊        | 215/1207 [2:05:14<8:58:40, 32.58s/it]{'loss': '0.2872', 'grad_norm': '0.4351', 'learning_rate': '9.839e-06', 'epoch': '0.174'}
+step=  215  loss=  0.2680  grad_norm= 0.554  lr=9.82e-06  spk_acc=1879/1918
+
                                                      

 18%|█▊        | 215/1207 [2:05:14<8:58:40, 32.58s/it]
 18%|█▊        | 216/1207 [2:05:45<8:50:47, 32.14s/it]
 18%|█▊        | 217/1207 [2:06:20<9:05:09, 33.04s/it]
 18%|█▊        | 218/1207 [2:06:55<9:11:07, 33.44s/it]
 18%|█▊        | 219/1207 [2:07:29<9:14:23, 33.67s/it]
 18%|█▊        | 220/1207 [2:08:05<9:25:20, 34.37s/it]{'loss': '0.268', 'grad_norm': '0.554', 'learning_rate': '9.82e-06', 'epoch': '0.1781'}
+step=  220  loss=  0.3168  grad_norm= 0.492  lr=9.80e-06  spk_acc=2478/2601
+
                                                      

 18%|█▊        | 220/1207 [2:08:05<9:25:20, 34.37s/it]
 18%|█▊        | 221/1207 [2:08:44<9:47:36, 35.76s/it]
 18%|█▊        | 222/1207 [2:09:23<10:02:46, 36.72s/it]
 18%|█▊        | 223/1207 [2:09:56<9:43:09, 35.56s/it] 
 19%|█▊        | 224/1207 [2:10:26<9:15:38, 33.92s/it]
 19%|█▊        | 225/1207 [2:10:59<9:14:32, 33.88s/it]{'loss': '0.3168', 'grad_norm': '0.4917', 'learning_rate': '9.8e-06', 'epoch': '0.1823'}
+step=  225  loss=  0.3675  grad_norm= 0.363  lr=9.78e-06  spk_acc=3339/3478
+
                                                      

 19%|█▊        | 225/1207 [2:10:59<9:14:32, 33.88s/it]
 19%|█▊        | 226/1207 [2:11:39<9:42:13, 35.61s/it]
 19%|█▉        | 227/1207 [2:12:11<9:25:17, 34.61s/it]
 19%|█▉        | 228/1207 [2:12:47<9:29:50, 34.92s/it]
 19%|█▉        | 229/1207 [2:13:20<9:17:57, 34.23s/it]
 19%|█▉        | 230/1207 [2:13:58<9:37:35, 35.47s/it]{'loss': '0.3675', 'grad_norm': '0.3629', 'learning_rate': '9.78e-06', 'epoch': '0.1864'}
+step=  230  loss=  0.3032  grad_norm= 0.597  lr=9.76e-06  spk_acc=2568/2705
+
                                                      

 19%|█▉        | 230/1207 [2:13:58<9:37:35, 35.47s/it]
 19%|█▉        | 231/1207 [2:14:38<9:59:52, 36.88s/it]
 19%|█▉        | 232/1207 [2:15:11<9:37:38, 35.55s/it]
 19%|█▉        | 233/1207 [2:15:46<9:38:43, 35.65s/it]
 19%|█▉        | 234/1207 [2:16:20<9:30:08, 35.16s/it]
 19%|█▉        | 235/1207 [2:16:56<9:29:28, 35.15s/it]{'loss': '0.3032', 'grad_norm': '0.5972', 'learning_rate': '9.758e-06', 'epoch': '0.1906'}
+step=  235  loss=  0.3212  grad_norm= 0.643  lr=9.74e-06  spk_acc=3011/3152
+
                                                      

 19%|█▉        | 235/1207 [2:16:56<9:29:28, 35.15s/it]
 20%|█▉        | 236/1207 [2:17:33<9:37:47, 35.70s/it]
 20%|█▉        | 237/1207 [2:18:18<10:25:35, 38.70s/it]
 20%|█▉        | 238/1207 [2:18:45<9:27:42, 35.15s/it] 
 20%|█▉        | 239/1207 [2:19:26<9:54:59, 36.88s/it]
 20%|█▉        | 240/1207 [2:19:59<9:34:05, 35.62s/it]{'loss': '0.3212', 'grad_norm': '0.6429', 'learning_rate': '9.735e-06', 'epoch': '0.1947'}
+step=  240  loss=  0.3546  grad_norm= 0.448  lr=9.71e-06  spk_acc=3329/3467
+
                                                      

 20%|█▉        | 240/1207 [2:19:59<9:34:05, 35.62s/it]
 20%|█▉        | 241/1207 [2:20:34<9:33:32, 35.62s/it]
 20%|██        | 242/1207 [2:21:06<9:14:33, 34.48s/it]
 20%|██        | 243/1207 [2:21:44<9:29:58, 35.48s/it]
 20%|██        | 244/1207 [2:22:19<9:26:26, 35.29s/it]
 20%|██        | 245/1207 [2:22:58<9:43:19, 36.38s/it]{'loss': '0.3546', 'grad_norm': '0.448', 'learning_rate': '9.712e-06', 'epoch': '0.1988'}
+step=  245  loss=  0.2691  grad_norm= 0.633  lr=9.69e-06  spk_acc=2017/2108
+
                                                      

 20%|██        | 245/1207 [2:22:58<9:43:19, 36.38s/it]
 20%|██        | 246/1207 [2:23:32<9:34:36, 35.88s/it]
 20%|██        | 247/1207 [2:24:05<9:16:35, 34.79s/it]
 21%|██        | 248/1207 [2:24:39<9:14:09, 34.67s/it]
 21%|██        | 249/1207 [2:25:14<9:15:19, 34.78s/it]
 21%|██        | 250/1207 [2:25:47<9:05:17, 34.19s/it]{'loss': '0.2691', 'grad_norm': '0.6335', 'learning_rate': '9.687e-06', 'epoch': '0.203'}
+step=  250  loss=  0.2699  grad_norm= 0.499  lr=9.66e-06  spk_acc=1822/1875
+
                                                      

 21%|██        | 250/1207 [2:25:47<9:05:17, 34.19s/it]
 21%|██        | 251/1207 [2:26:21<9:05:07, 34.21s/it]
 21%|██        | 252/1207 [2:26:38<7:41:23, 28.99s/it]
 21%|██        | 253/1207 [2:27:13<8:09:00, 30.76s/it]
 21%|██        | 254/1207 [2:27:51<8:41:39, 32.84s/it]
 21%|██        | 255/1207 [2:28:34<9:29:08, 35.87s/it]{'loss': '0.2699', 'grad_norm': '0.4988', 'learning_rate': '9.661e-06', 'epoch': '0.2071'}
+step=  255  loss=  0.3373  grad_norm= 0.497  lr=9.63e-06  spk_acc=2752/2879
+
                                                      

 21%|██        | 255/1207 [2:28:34<9:29:08, 35.87s/it]
 21%|██        | 256/1207 [2:29:17<10:02:31, 38.01s/it]
 21%|██▏       | 257/1207 [2:29:52<9:51:34, 37.36s/it] 
 21%|██▏       | 258/1207 [2:30:27<9:38:37, 36.58s/it]
 21%|██▏       | 259/1207 [2:31:03<9:34:24, 36.35s/it]
 22%|██▏       | 260/1207 [2:31:41<9:40:14, 36.76s/it]{'loss': '0.3373', 'grad_norm': '0.4971', 'learning_rate': '9.634e-06', 'epoch': '0.2113'}
+step=  260  loss=  0.3382  grad_norm= 0.578  lr=9.61e-06  spk_acc=3011/3224
+
                                                      

 22%|██▏       | 260/1207 [2:31:41<9:40:14, 36.76s/it]
 22%|██▏       | 261/1207 [2:32:15<9:29:54, 36.15s/it]
 22%|██▏       | 262/1207 [2:32:51<9:27:20, 36.02s/it]
 22%|██▏       | 263/1207 [2:33:29<9:34:05, 36.49s/it]
 22%|██▏       | 264/1207 [2:34:02<9:16:06, 35.38s/it]
 22%|██▏       | 265/1207 [2:34:37<9:13:36, 35.26s/it]{'loss': '0.3382', 'grad_norm': '0.5779', 'learning_rate': '9.607e-06', 'epoch': '0.2154'}
+step=  265  loss=  0.2622  grad_norm= 0.581  lr=9.58e-06  spk_acc=2428/2463
+
                                                      

 22%|██▏       | 265/1207 [2:34:37<9:13:36, 35.26s/it]
 22%|██▏       | 266/1207 [2:35:13<9:19:16, 35.66s/it]
 22%|██▏       | 267/1207 [2:35:47<9:08:03, 34.98s/it]
 22%|██▏       | 268/1207 [2:36:23<9:12:54, 35.33s/it]
 22%|██▏       | 269/1207 [2:37:03<9:33:34, 36.69s/it]
 22%|██▏       | 270/1207 [2:37:35<9:12:19, 35.37s/it]{'loss': '0.2622', 'grad_norm': '0.5814', 'learning_rate': '9.578e-06', 'epoch': '0.2196'}
+step=  270  loss=  0.3134  grad_norm= 0.551  lr=9.55e-06  spk_acc=2699/2849
+
                                                      

 22%|██▏       | 270/1207 [2:37:35<9:12:19, 35.37s/it]
 22%|██▏       | 271/1207 [2:38:09<9:07:10, 35.08s/it]
 23%|██▎       | 272/1207 [2:38:43<9:02:51, 34.84s/it]
 23%|██▎       | 273/1207 [2:39:14<8:40:03, 33.41s/it]
 23%|██▎       | 274/1207 [2:39:48<8:42:55, 33.63s/it]
 23%|██▎       | 275/1207 [2:40:22<8:43:16, 33.69s/it]{'loss': '0.3134', 'grad_norm': '0.5512', 'learning_rate': '9.549e-06', 'epoch': '0.2237'}
+step=  275  loss=  0.2432  grad_norm= 0.517  lr=9.52e-06  spk_acc=1745/1796
+
                                                      

 23%|██▎       | 275/1207 [2:40:22<8:43:16, 33.69s/it]
 23%|██▎       | 276/1207 [2:41:03<9:21:02, 36.16s/it]
 23%|██▎       | 277/1207 [2:41:37<9:08:22, 35.38s/it]
 23%|██▎       | 278/1207 [2:42:11<9:02:13, 35.02s/it]
 23%|██▎       | 279/1207 [2:42:48<9:08:52, 35.49s/it]
 23%|██▎       | 280/1207 [2:43:27<9:25:28, 36.60s/it]{'loss': '0.2432', 'grad_norm': '0.5168', 'learning_rate': '9.518e-06', 'epoch': '0.2278'}
+step=  280  loss=  0.3253  grad_norm= 0.493  lr=9.49e-06  spk_acc=2779/2925
+
                                                      

 23%|██▎       | 280/1207 [2:43:27<9:25:28, 36.60s/it]
 23%|██▎       | 281/1207 [2:44:04<9:25:45, 36.66s/it]
 23%|██▎       | 282/1207 [2:44:49<10:07:09, 39.38s/it]
 23%|██▎       | 283/1207 [2:45:21<9:32:05, 37.15s/it] 
 24%|██▎       | 284/1207 [2:45:59<9:32:40, 37.23s/it]
 24%|██▎       | 285/1207 [2:46:30<9:06:19, 35.55s/it]{'loss': '0.3253', 'grad_norm': '0.493', 'learning_rate': '9.487e-06', 'epoch': '0.232'}
+step=  285  loss=  0.3033  grad_norm= 0.461  lr=9.45e-06  spk_acc=2787/2889
+
                                                      

 24%|██▎       | 285/1207 [2:46:30<9:06:19, 35.55s/it]
 24%|██▎       | 286/1207 [2:47:12<9:32:32, 37.30s/it]
 24%|██▍       | 287/1207 [2:47:47<9:20:32, 36.56s/it]
 24%|██▍       | 288/1207 [2:48:18<8:57:18, 35.08s/it]
 24%|██▍       | 289/1207 [2:48:58<9:17:56, 36.47s/it]
 24%|██▍       | 290/1207 [2:49:37<9:28:42, 37.21s/it]{'loss': '0.3033', 'grad_norm': '0.461', 'learning_rate': '9.454e-06', 'epoch': '0.2361'}
+step=  290  loss=  0.2892  grad_norm= 0.292  lr=9.42e-06  spk_acc=2794/2844
+
                                                      

 24%|██▍       | 290/1207 [2:49:37<9:28:42, 37.21s/it]
 24%|██▍       | 291/1207 [2:50:06<8:50:14, 34.73s/it]
 24%|██▍       | 292/1207 [2:50:39<8:40:41, 34.14s/it]
 24%|██▍       | 293/1207 [2:51:12<8:35:25, 33.84s/it]
 24%|██▍       | 294/1207 [2:51:49<8:49:53, 34.82s/it]
 24%|██▍       | 295/1207 [2:52:25<8:55:54, 35.26s/it]{'loss': '0.2892', 'grad_norm': '0.2923', 'learning_rate': '9.421e-06', 'epoch': '0.2403'}
+step=  295  loss=  0.2719  grad_norm= 0.376  lr=9.39e-06  spk_acc=2604/2650
+
                                                      

 24%|██▍       | 295/1207 [2:52:25<8:55:54, 35.26s/it]
 25%|██▍       | 296/1207 [2:52:58<8:44:39, 34.55s/it]
 25%|██▍       | 297/1207 [2:53:27<8:17:31, 32.80s/it]
 25%|██▍       | 298/1207 [2:54:02<8:27:33, 33.50s/it]
 25%|██▍       | 299/1207 [2:54:39<8:44:14, 34.64s/it]
 25%|██▍       | 300/1207 [2:55:17<8:56:59, 35.52s/it]{'loss': '0.2719', 'grad_norm': '0.3756', 'learning_rate': '9.387e-06', 'epoch': '0.2444'}
+step=  300  loss=  0.3238  grad_norm= 0.345  lr=9.35e-06  spk_acc=3143/3291
+
                                                      

 25%|██▍       | 300/1207 [2:55:17<8:56:59, 35.52s/it]
 25%|██▍       | 301/1207 [2:55:49<8:40:57, 34.50s/it]
 25%|██▌       | 302/1207 [2:56:28<8:59:44, 35.78s/it]
 25%|██▌       | 303/1207 [2:57:07<9:14:48, 36.82s/it]
 25%|██▌       | 304/1207 [2:57:47<9:28:49, 37.80s/it]
 25%|██▌       | 305/1207 [2:58:16<8:46:35, 35.03s/it]{'loss': '0.3238', 'grad_norm': '0.3454', 'learning_rate': '9.352e-06', 'epoch': '0.2486'}
+step=  305  loss=  0.3261  grad_norm= 0.389  lr=9.32e-06  spk_acc=3202/3454
+
                                                      

 25%|██▌       | 305/1207 [2:58:16<8:46:35, 35.03s/it]
 25%|██▌       | 306/1207 [2:58:47<8:29:44, 33.94s/it]
 25%|██▌       | 307/1207 [2:59:25<8:45:24, 35.03s/it]
 26%|██▌       | 308/1207 [2:59:57<8:33:35, 34.28s/it]
 26%|██▌       | 309/1207 [3:00:31<8:31:58, 34.21s/it]
 26%|██▌       | 310/1207 [3:01:06<8:32:22, 34.27s/it]{'loss': '0.3261', 'grad_norm': '0.3886', 'learning_rate': '9.316e-06', 'epoch': '0.2527'}
+step=  310  loss=  0.3176  grad_norm= 0.479  lr=9.28e-06  spk_acc=2279/2370
+
                                                      

 26%|██▌       | 310/1207 [3:01:06<8:32:22, 34.27s/it]
 26%|██▌       | 311/1207 [3:01:42<8:40:20, 34.84s/it]
 26%|██▌       | 312/1207 [3:02:08<8:03:13, 32.39s/it]
 26%|██▌       | 313/1207 [3:02:43<8:11:12, 32.97s/it]
 26%|██▌       | 314/1207 [3:03:23<8:43:54, 35.20s/it]
 26%|██▌       | 315/1207 [3:03:56<8:33:31, 34.54s/it]{'loss': '0.3176', 'grad_norm': '0.4789', 'learning_rate': '9.279e-06', 'epoch': '0.2568'}
+step=  315  loss=  0.3328  grad_norm= 0.555  lr=9.24e-06  spk_acc=3043/3203
+
                                                      

 26%|██▌       | 315/1207 [3:03:56<8:33:31, 34.54s/it]
 26%|██▌       | 316/1207 [3:04:31<8:36:09, 34.76s/it]
 26%|██▋       | 317/1207 [3:05:11<8:58:50, 36.33s/it]
 26%|██▋       | 318/1207 [3:05:57<9:38:21, 39.03s/it]
 26%|██▋       | 319/1207 [3:06:34<9:29:02, 38.45s/it]
 27%|██▋       | 320/1207 [3:07:04<8:52:37, 36.03s/it]{'loss': '0.3328', 'grad_norm': '0.5554', 'learning_rate': '9.241e-06', 'epoch': '0.261'}
+step=  320  loss=  0.3482  grad_norm= 0.547  lr=9.20e-06  spk_acc=4117/4370
+
                                                      

 27%|██▋       | 320/1207 [3:07:04<8:52:37, 36.03s/it]
 27%|██▋       | 321/1207 [3:07:38<8:44:06, 35.49s/it]
 27%|██▋       | 322/1207 [3:08:10<8:26:10, 34.32s/it]
 27%|██▋       | 323/1207 [3:08:40<8:05:21, 32.94s/it]
 27%|██▋       | 324/1207 [3:09:11<7:56:35, 32.38s/it]
 27%|██▋       | 325/1207 [3:09:45<8:05:36, 33.03s/it]{'loss': '0.3482', 'grad_norm': '0.5473', 'learning_rate': '9.202e-06', 'epoch': '0.2651'}
+step=  325  loss=  0.2724  grad_norm= 0.651  lr=9.16e-06  spk_acc=2283/2352
+
                                                      

 27%|██▋       | 325/1207 [3:09:45<8:05:36, 33.03s/it]
 27%|██▋       | 326/1207 [3:10:13<7:39:14, 31.28s/it]
 27%|██▋       | 327/1207 [3:10:46<7:49:27, 32.01s/it]
 27%|██▋       | 328/1207 [3:11:25<8:18:42, 34.04s/it]
 27%|██▋       | 329/1207 [3:12:06<8:49:57, 36.22s/it]
 27%|██▋       | 330/1207 [3:12:41<8:40:29, 35.61s/it]{'loss': '0.2724', 'grad_norm': '0.6511', 'learning_rate': '9.162e-06', 'epoch': '0.2693'}
+step=  330  loss=  0.3448  grad_norm= 0.521  lr=9.12e-06  spk_acc=2873/3069
+
                                                      

 27%|██▋       | 330/1207 [3:12:41<8:40:29, 35.61s/it]
 27%|██▋       | 331/1207 [3:13:11<8:18:59, 34.18s/it]
 28%|██▊       | 332/1207 [3:13:46<8:18:41, 34.20s/it]
 28%|██▊       | 333/1207 [3:14:20<8:18:10, 34.20s/it]
 28%|██▊       | 334/1207 [3:15:00<8:41:46, 35.86s/it]
 28%|██▊       | 335/1207 [3:15:41<9:03:18, 37.38s/it]{'loss': '0.3448', 'grad_norm': '0.5212', 'learning_rate': '9.122e-06', 'epoch': '0.2734'}
+step=  335  loss=  0.3662  grad_norm= 0.878  lr=9.08e-06  spk_acc=3111/3239
+
                                                      

 28%|██▊       | 335/1207 [3:15:41<9:03:18, 37.38s/it]
 28%|██▊       | 336/1207 [3:16:15<8:49:51, 36.50s/it]
 28%|██▊       | 337/1207 [3:16:52<8:51:23, 36.65s/it]
 28%|██▊       | 338/1207 [3:17:25<8:35:05, 35.56s/it]
 28%|██▊       | 339/1207 [3:17:57<8:20:44, 34.61s/it]
 28%|██▊       | 340/1207 [3:18:30<8:13:12, 34.13s/it]{'loss': '0.3662', 'grad_norm': '0.8779', 'learning_rate': '9.08e-06', 'epoch': '0.2775'}
+step=  340  loss=  0.2668  grad_norm= 0.870  lr=9.04e-06  spk_acc=2161/2237
+
                                                      

 28%|██▊       | 340/1207 [3:18:30<8:13:12, 34.13s/it]
 28%|██▊       | 341/1207 [3:19:07<8:24:31, 34.96s/it]
 28%|██▊       | 342/1207 [3:19:37<8:02:16, 33.45s/it]
 28%|██▊       | 343/1207 [3:20:13<8:13:34, 34.28s/it]
 29%|██▊       | 344/1207 [3:20:46<8:05:49, 33.78s/it]
 29%|██▊       | 345/1207 [3:21:21<8:11:46, 34.23s/it]{'loss': '0.2668', 'grad_norm': '0.8696', 'learning_rate': '9.038e-06', 'epoch': '0.2817'}
+step=  345  loss=  0.2605  grad_norm= 0.447  lr=9.00e-06  spk_acc=2332/2359
+
                                                      

 29%|██▊       | 345/1207 [3:21:21<8:11:46, 34.23s/it]
 29%|██▊       | 346/1207 [3:21:59<8:26:29, 35.30s/it]
 29%|██▊       | 347/1207 [3:22:37<8:38:57, 36.21s/it]
 29%|██▉       | 348/1207 [3:23:07<8:09:31, 34.19s/it]
 29%|██▉       | 349/1207 [3:23:38<7:55:15, 33.24s/it]
 29%|██▉       | 350/1207 [3:24:10<7:49:13, 32.85s/it]{'loss': '0.2605', 'grad_norm': '0.4467', 'learning_rate': '8.995e-06', 'epoch': '0.2858'}
+step=  350  loss=  0.2614  grad_norm= 0.429  lr=8.95e-06  spk_acc=2337/2415
+
                                                      

 29%|██▉       | 350/1207 [3:24:10<7:49:13, 32.85s/it]
 29%|██▉       | 351/1207 [3:24:42<7:44:24, 32.55s/it]
 29%|██▉       | 352/1207 [3:25:16<7:52:18, 33.14s/it]
 29%|██▉       | 353/1207 [3:25:45<7:33:13, 31.84s/it]
 29%|██▉       | 354/1207 [3:26:22<7:52:35, 33.24s/it]
 29%|██▉       | 355/1207 [3:26:58<8:05:13, 34.17s/it]{'loss': '0.2614', 'grad_norm': '0.4295', 'learning_rate': '8.951e-06', 'epoch': '0.29'}
+step=  355  loss=  0.2584  grad_norm= 0.386  lr=8.91e-06  spk_acc=2113/2144
+
                                                      

 29%|██▉       | 355/1207 [3:26:58<8:05:13, 34.17s/it]
 29%|██▉       | 356/1207 [3:27:35<8:15:57, 34.97s/it]
 30%|██▉       | 357/1207 [3:28:07<8:05:42, 34.29s/it]
 30%|██▉       | 358/1207 [3:28:45<8:18:32, 35.23s/it]
 30%|██▉       | 359/1207 [3:29:20<8:17:42, 35.22s/it]
 30%|██▉       | 360/1207 [3:29:58<8:30:31, 36.16s/it]{'loss': '0.2584', 'grad_norm': '0.3859', 'learning_rate': '8.907e-06', 'epoch': '0.2941'}
+step=  360  loss=  0.3345  grad_norm= 0.644  lr=8.86e-06  spk_acc=3097/3272
+
                                                      

 30%|██▉       | 360/1207 [3:29:58<8:30:31, 36.16s/it]
 30%|██▉       | 361/1207 [3:30:36<8:34:47, 36.51s/it]
 30%|██▉       | 362/1207 [3:31:16<8:48:56, 37.56s/it]
 30%|███       | 363/1207 [3:31:47<8:21:33, 35.66s/it]
 30%|███       | 364/1207 [3:32:17<7:57:06, 33.96s/it]
 30%|███       | 365/1207 [3:32:53<8:03:24, 34.45s/it]{'loss': '0.3345', 'grad_norm': '0.6442', 'learning_rate': '8.861e-06', 'epoch': '0.2983'}
+step=  365  loss=  0.2878  grad_norm= 0.414  lr=8.81e-06  spk_acc=2618/2718
+
                                                      

 30%|███       | 365/1207 [3:32:53<8:03:24, 34.45s/it]
 30%|███       | 366/1207 [3:33:27<8:02:43, 34.44s/it]
 30%|███       | 367/1207 [3:33:59<7:53:42, 33.84s/it]
 30%|███       | 368/1207 [3:34:30<7:40:48, 32.95s/it]
 31%|███       | 369/1207 [3:35:06<7:50:18, 33.67s/it]
 31%|███       | 370/1207 [3:35:39<7:48:38, 33.59s/it]{'loss': '0.2878', 'grad_norm': '0.4142', 'learning_rate': '8.815e-06', 'epoch': '0.3024'}
+step=  370  loss=  0.2929  grad_norm= 0.467  lr=8.77e-06  spk_acc=2154/2197
+
                                                      

 31%|███       | 370/1207 [3:35:39<7:48:38, 33.59s/it]
 31%|███       | 371/1207 [3:36:12<7:46:58, 33.51s/it]
 31%|███       | 372/1207 [3:36:44<7:36:58, 32.84s/it]
 31%|███       | 373/1207 [3:37:19<7:48:48, 33.73s/it]
 31%|███       | 374/1207 [3:37:49<7:32:52, 32.62s/it]
 31%|███       | 375/1207 [3:38:23<7:37:02, 32.96s/it]{'loss': '0.2929', 'grad_norm': '0.4671', 'learning_rate': '8.768e-06', 'epoch': '0.3065'}
+step=  375  loss=  0.2993  grad_norm= 1.141  lr=8.72e-06  spk_acc=2211/2298
+
                                                      

 31%|███       | 375/1207 [3:38:23<7:37:02, 32.96s/it]
 31%|███       | 376/1207 [3:39:00<7:53:23, 34.18s/it]
 31%|███       | 377/1207 [3:39:36<7:59:56, 34.69s/it]
 31%|███▏      | 378/1207 [3:40:09<7:52:38, 34.21s/it]
 31%|███▏      | 379/1207 [3:40:42<7:45:13, 33.71s/it]
 31%|███▏      | 380/1207 [3:41:14<7:39:30, 33.34s/it]{'loss': '0.2993', 'grad_norm': '1.141', 'learning_rate': '8.72e-06', 'epoch': '0.3107'}
+step=  380  loss=  0.3301  grad_norm= 0.935  lr=8.67e-06  spk_acc=3051/3181
+
                                                      

 31%|███▏      | 380/1207 [3:41:14<7:39:30, 33.34s/it]
 32%|███▏      | 381/1207 [3:41:53<8:00:08, 34.88s/it]
 32%|███▏      | 382/1207 [3:42:32<8:18:38, 36.26s/it]
 32%|███▏      | 383/1207 [3:42:57<7:32:13, 32.93s/it]
 32%|███▏      | 384/1207 [3:43:26<7:14:36, 31.69s/it]
 32%|███▏      | 385/1207 [3:44:00<7:22:46, 32.32s/it]{'loss': '0.3301', 'grad_norm': '0.9348', 'learning_rate': '8.671e-06', 'epoch': '0.3148'}
+step=  385  loss=  0.3574  grad_norm= 0.492  lr=8.62e-06  spk_acc=2708/2850
+
                                                      

 32%|███▏      | 385/1207 [3:44:00<7:22:46, 32.32s/it]
 32%|███▏      | 386/1207 [3:44:32<7:21:26, 32.26s/it]
 32%|███▏      | 387/1207 [3:45:06<7:29:22, 32.88s/it]
 32%|███▏      | 388/1207 [3:45:40<7:33:36, 33.23s/it]
 32%|███▏      | 389/1207 [3:46:17<7:46:13, 34.20s/it]
 32%|███▏      | 390/1207 [3:46:51<7:44:03, 34.08s/it]{'loss': '0.3574', 'grad_norm': '0.4922', 'learning_rate': '8.621e-06', 'epoch': '0.319'}
+step=  390  loss=  0.2930  grad_norm= 0.479  lr=8.57e-06  spk_acc=2189/2252
+
                                                      

 32%|███▏      | 390/1207 [3:46:51<7:44:03, 34.08s/it]
 32%|███▏      | 391/1207 [3:47:27<7:52:28, 34.74s/it]
 32%|███▏      | 392/1207 [3:48:05<8:05:29, 35.74s/it]
 33%|███▎      | 393/1207 [3:48:38<7:51:41, 34.77s/it]
 33%|███▎      | 394/1207 [3:49:12<7:49:02, 34.62s/it]
 33%|███▎      | 395/1207 [3:49:48<7:56:14, 35.19s/it]{'loss': '0.293', 'grad_norm': '0.4788', 'learning_rate': '8.571e-06', 'epoch': '0.3231'}
+step=  395  loss=  0.2775  grad_norm= 0.443  lr=8.52e-06  spk_acc=2939/3068
+
                                                      

 33%|███▎      | 395/1207 [3:49:48<7:56:14, 35.19s/it]
 33%|███▎      | 396/1207 [3:50:30<8:20:51, 37.06s/it]
 33%|███▎      | 397/1207 [3:51:14<8:49:48, 39.24s/it]
 33%|███▎      | 398/1207 [3:51:50<8:35:00, 38.20s/it]
 33%|███▎      | 399/1207 [3:52:28<8:34:26, 38.20s/it]
 33%|███▎      | 400/1207 [3:53:02<8:16:06, 36.89s/it]{'loss': '0.2775', 'grad_norm': '0.4427', 'learning_rate': '8.52e-06', 'epoch': '0.3273'}
+step=  400  loss=  0.3408  grad_norm= 0.581  lr=8.47e-06  spk_acc=3549/3704
+
                                                      

 33%|███▎      | 400/1207 [3:53:02<8:16:06, 36.89s/it]{'loss': '0.3408', 'grad_norm': '0.5809', 'learning_rate': '8.468e-06', 'epoch': '0.3314'}
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.04s/it]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.05s/it]
+/wangshuai/moss/MOSS_Speaker-RoPE/moss_speaker_rope/wespeaker_redimnet/tfmel.py:296: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+
 33%|███▎      | 401/1207 [3:53:39<8:16:39, 36.97s/it]
 33%|███▎      | 402/1207 [3:54:09<7:48:51, 34.95s/it]
 33%|███▎      | 403/1207 [3:54:40<7:31:09, 33.67s/it]
 33%|███▎      | 404/1207 [3:55:11<7:18:51, 32.79s/it]
 34%|███▎      | 405/1207 [3:55:43<7:17:10, 32.71s/it]step=  405  loss=  0.2912  grad_norm= 0.490  lr=8.42e-06  spk_acc=1914/1972
+
                                                      

 34%|███▎      | 405/1207 [3:55:43<7:17:10, 32.71s/it]
 34%|███▎      | 406/1207 [3:56:18<7:23:57, 33.26s/it]
 34%|███▎      | 407/1207 [3:56:58<7:52:22, 35.43s/it]
 34%|███▍      | 408/1207 [3:57:35<7:56:36, 35.79s/it]
 34%|███▍      | 409/1207 [3:58:05<7:31:38, 33.96s/it]
 34%|███▍      | 410/1207 [3:58:39<7:33:55, 34.17s/it]{'loss': '0.2912', 'grad_norm': '0.4902', 'learning_rate': '8.416e-06', 'epoch': '0.3355'}
+step=  410  loss=  0.2749  grad_norm= 0.402  lr=8.36e-06  spk_acc=2864/3006
+
                                                      

 34%|███▍      | 410/1207 [3:58:39<7:33:55, 34.17s/it]
 34%|███▍      | 411/1207 [3:59:15<7:37:53, 34.51s/it]
 34%|███▍      | 412/1207 [3:59:52<7:47:57, 35.32s/it]
 34%|███▍      | 413/1207 [4:00:26<7:41:23, 34.87s/it]
 34%|███▍      | 414/1207 [4:01:07<8:07:34, 36.89s/it]
 34%|███▍      | 415/1207 [4:01:37<7:39:45, 34.83s/it]{'loss': '0.2749', 'grad_norm': '0.402', 'learning_rate': '8.363e-06', 'epoch': '0.3397'}
+step=  415  loss=  0.3353  grad_norm= 0.557  lr=8.31e-06  spk_acc=3409/3506
+
                                                      

 34%|███▍      | 415/1207 [4:01:37<7:39:45, 34.83s/it]
 34%|███▍      | 416/1207 [4:02:11<7:35:04, 34.52s/it]
 35%|███▍      | 417/1207 [4:02:46<7:36:15, 34.65s/it]
 35%|███▍      | 418/1207 [4:03:21<7:37:53, 34.82s/it]
 35%|███▍      | 419/1207 [4:03:51<7:15:54, 33.19s/it]
 35%|███▍      | 420/1207 [4:04:22<7:06:59, 32.55s/it]{'loss': '0.3353', 'grad_norm': '0.5575', 'learning_rate': '8.309e-06', 'epoch': '0.3438'}
+step=  420  loss=  0.3235  grad_norm= 0.723  lr=8.25e-06  spk_acc=2067/2180
+
                                                      

 35%|███▍      | 420/1207 [4:04:22<7:06:59, 32.55s/it]
 35%|███▍      | 421/1207 [4:05:01<7:32:03, 34.51s/it]
 35%|███▍      | 422/1207 [4:05:35<7:31:57, 34.54s/it]
 35%|███▌      | 423/1207 [4:06:13<7:44:54, 35.58s/it]
 35%|███▌      | 424/1207 [4:06:45<7:29:02, 34.41s/it]
 35%|███▌      | 425/1207 [4:07:15<7:12:23, 33.18s/it]{'loss': '0.3235', 'grad_norm': '0.723', 'learning_rate': '8.254e-06', 'epoch': '0.348'}
+step=  425  loss=  0.3235  grad_norm= 0.641  lr=8.20e-06  spk_acc=2462/2606
+
                                                      

 35%|███▌      | 425/1207 [4:07:15<7:12:23, 33.18s/it]
 35%|███▌      | 426/1207 [4:07:50<7:17:23, 33.60s/it]
 35%|███▌      | 427/1207 [4:08:29<7:37:39, 35.20s/it]
 35%|███▌      | 428/1207 [4:09:00<7:21:46, 34.03s/it]
 36%|███▌      | 429/1207 [4:09:35<7:24:12, 34.26s/it]
 36%|███▌      | 430/1207 [4:10:12<7:32:39, 34.95s/it]{'loss': '0.3235', 'grad_norm': '0.641', 'learning_rate': '8.199e-06', 'epoch': '0.3521'}
+step=  430  loss=  0.2962  grad_norm= 0.689  lr=8.14e-06  spk_acc=2044/2171
+
                                                      

 36%|███▌      | 430/1207 [4:10:12<7:32:39, 34.95s/it]
 36%|███▌      | 431/1207 [4:10:53<7:57:01, 36.88s/it]
 36%|███▌      | 432/1207 [4:11:27<7:45:48, 36.06s/it]
 36%|███▌      | 433/1207 [4:11:53<7:06:49, 33.09s/it]
 36%|███▌      | 434/1207 [4:12:20<6:41:16, 31.15s/it]
 36%|███▌      | 435/1207 [4:12:51<6:40:19, 31.11s/it]{'loss': '0.2962', 'grad_norm': '0.6892', 'learning_rate': '8.143e-06', 'epoch': '0.3563'}
+step=  435  loss=  0.3264  grad_norm= 0.499  lr=8.09e-06  spk_acc=2524/2586
+
                                                      

 36%|███▌      | 435/1207 [4:12:51<6:40:19, 31.11s/it]
 36%|███▌      | 436/1207 [4:13:25<6:51:01, 31.99s/it]
 36%|███▌      | 437/1207 [4:14:04<7:19:04, 34.21s/it]
 36%|███▋      | 438/1207 [4:14:42<7:30:32, 35.15s/it]
 36%|███▋      | 439/1207 [4:15:16<7:27:37, 34.97s/it]
 36%|███▋      | 440/1207 [4:15:45<7:04:30, 33.21s/it]{'loss': '0.3264', 'grad_norm': '0.4989', 'learning_rate': '8.087e-06', 'epoch': '0.3604'}
+step=  440  loss=  0.3216  grad_norm= 0.535  lr=8.03e-06  spk_acc=2667/2871
+
                                                      

 36%|███▋      | 440/1207 [4:15:45<7:04:30, 33.21s/it]
 37%|███▋      | 441/1207 [4:16:17<6:57:09, 32.68s/it]
 37%|███▋      | 442/1207 [4:16:51<7:04:05, 33.26s/it]
 37%|███▋      | 443/1207 [4:17:32<7:32:30, 35.54s/it]
 37%|███▋      | 444/1207 [4:18:01<7:07:02, 33.58s/it]
 37%|███▋      | 445/1207 [4:18:30<6:48:42, 32.18s/it]{'loss': '0.3216', 'grad_norm': '0.5351', 'learning_rate': '8.029e-06', 'epoch': '0.3645'}
+step=  445  loss=  0.3500  grad_norm= 0.588  lr=7.97e-06  spk_acc=2745/2872
+
                                                      

 37%|███▋      | 445/1207 [4:18:30<6:48:42, 32.18s/it]
 37%|███▋      | 446/1207 [4:19:04<6:54:43, 32.70s/it]
 37%|███▋      | 447/1207 [4:19:41<7:09:07, 33.88s/it]
 37%|███▋      | 448/1207 [4:20:19<7:25:04, 35.18s/it]
 37%|███▋      | 449/1207 [4:20:56<7:32:08, 35.79s/it]
 37%|███▋      | 450/1207 [4:21:30<7:23:54, 35.18s/it]{'loss': '0.35', 'grad_norm': '0.5879', 'learning_rate': '7.972e-06', 'epoch': '0.3687'}
+step=  450  loss=  0.3092  grad_norm= 0.670  lr=7.91e-06  spk_acc=2775/2945
+
                                                      

 37%|███▋      | 450/1207 [4:21:30<7:23:54, 35.18s/it]
 37%|███▋      | 451/1207 [4:22:05<7:23:07, 35.17s/it]
 37%|███▋      | 452/1207 [4:22:36<7:08:26, 34.05s/it]
 38%|███▊      | 453/1207 [4:23:03<6:38:47, 31.73s/it]
 38%|███▊      | 454/1207 [4:23:43<7:12:01, 34.42s/it]
 38%|███▊      | 455/1207 [4:24:25<7:39:45, 36.68s/it]{'loss': '0.3092', 'grad_norm': '0.6696', 'learning_rate': '7.913e-06', 'epoch': '0.3728'}
+step=  455  loss=  0.3417  grad_norm= 0.679  lr=7.85e-06  spk_acc=3165/3309
+
                                                      

 38%|███▊      | 455/1207 [4:24:25<7:39:45, 36.68s/it]
 38%|███▊      | 456/1207 [4:24:54<7:09:57, 34.35s/it]
 38%|███▊      | 457/1207 [4:25:29<7:11:11, 34.50s/it]
 38%|███▊      | 458/1207 [4:26:01<7:00:58, 33.72s/it]
 38%|███▊      | 459/1207 [4:26:33<6:54:06, 33.22s/it]
 38%|███▊      | 460/1207 [4:27:06<6:51:51, 33.08s/it]{'loss': '0.3417', 'grad_norm': '0.6794', 'learning_rate': '7.854e-06', 'epoch': '0.377'}
+step=  460  loss=  0.2695  grad_norm= 0.362  lr=7.79e-06  spk_acc=2185/2287
+
                                                      

 38%|███▊      | 460/1207 [4:27:06<6:51:51, 33.08s/it]
 38%|███▊      | 461/1207 [4:27:37<6:43:14, 32.43s/it]
 38%|███▊      | 462/1207 [4:28:11<6:50:34, 33.07s/it]
 38%|███▊      | 463/1207 [4:28:46<6:55:04, 33.47s/it]
 38%|███▊      | 464/1207 [4:29:07<6:07:20, 29.66s/it]
 39%|███▊      | 465/1207 [4:29:39<6:18:06, 30.58s/it]{'loss': '0.2695', 'grad_norm': '0.3616', 'learning_rate': '7.794e-06', 'epoch': '0.3811'}
+step=  465  loss=  0.3314  grad_norm= 0.470  lr=7.73e-06  spk_acc=2128/2162
+
                                                      

 39%|███▊      | 465/1207 [4:29:39<6:18:06, 30.58s/it]
 39%|███▊      | 466/1207 [4:30:17<6:42:51, 32.62s/it]
 39%|███▊      | 467/1207 [4:30:53<6:57:29, 33.85s/it]
 39%|███▉      | 468/1207 [4:31:25<6:50:23, 33.32s/it]
 39%|███▉      | 469/1207 [4:32:01<6:58:03, 33.99s/it]
 39%|███▉      | 470/1207 [4:32:36<7:00:12, 34.21s/it]{'loss': '0.3314', 'grad_norm': '0.47', 'learning_rate': '7.734e-06', 'epoch': '0.3853'}
+step=  470  loss=  0.3427  grad_norm= 0.453  lr=7.67e-06  spk_acc=3094/3298
+
                                                      

 39%|███▉      | 470/1207 [4:32:36<7:00:12, 34.21s/it]
 39%|███▉      | 471/1207 [4:33:08<6:52:09, 33.60s/it]
 39%|███▉      | 472/1207 [4:33:38<6:39:53, 32.64s/it]
 39%|███▉      | 473/1207 [4:34:12<6:41:52, 32.85s/it]
 39%|███▉      | 474/1207 [4:34:46<6:45:22, 33.18s/it]
 39%|███▉      | 475/1207 [4:35:16<6:35:05, 32.39s/it]{'loss': '0.3427', 'grad_norm': '0.4527', 'learning_rate': '7.673e-06', 'epoch': '0.3894'}
+step=  475  loss=  0.2484  grad_norm= 0.484  lr=7.61e-06  spk_acc=1973/2018
+
                                                      

 39%|███▉      | 475/1207 [4:35:16<6:35:05, 32.39s/it]
 39%|███▉      | 476/1207 [4:35:50<6:39:32, 32.79s/it]
 40%|███▉      | 477/1207 [4:36:23<6:41:57, 33.04s/it]
 40%|███▉      | 478/1207 [4:36:55<6:36:24, 32.63s/it]
 40%|███▉      | 479/1207 [4:37:28<6:35:22, 32.59s/it]
 40%|███▉      | 480/1207 [4:38:06<6:57:45, 34.48s/it]{'loss': '0.2484', 'grad_norm': '0.484', 'learning_rate': '7.612e-06', 'epoch': '0.3935'}
+step=  480  loss=  0.2822  grad_norm= 0.300  lr=7.55e-06  spk_acc=2233/2327
+
                                                      

 40%|███▉      | 480/1207 [4:38:06<6:57:45, 34.48s/it]
 40%|███▉      | 481/1207 [4:38:41<6:58:09, 34.56s/it]
 40%|███▉      | 482/1207 [4:39:15<6:55:22, 34.38s/it]
 40%|████      | 483/1207 [4:39:43<6:31:51, 32.47s/it]
 40%|████      | 484/1207 [4:40:24<7:02:00, 35.02s/it]
 40%|████      | 485/1207 [4:41:04<7:20:09, 36.58s/it]{'loss': '0.2822', 'grad_norm': '0.2996', 'learning_rate': '7.55e-06', 'epoch': '0.3977'}
+step=  485  loss=  0.2629  grad_norm= 0.408  lr=7.49e-06  spk_acc=3327/3380
+
                                                      

 40%|████      | 485/1207 [4:41:04<7:20:09, 36.58s/it]
 40%|████      | 486/1207 [4:41:38<7:09:23, 35.73s/it]
 40%|████      | 487/1207 [4:42:10<6:54:37, 34.55s/it]
 40%|████      | 488/1207 [4:42:46<6:59:57, 35.05s/it]
 41%|████      | 489/1207 [4:43:20<6:56:13, 34.78s/it]
 41%|████      | 490/1207 [4:43:52<6:44:34, 33.86s/it]{'loss': '0.2629', 'grad_norm': '0.4082', 'learning_rate': '7.487e-06', 'epoch': '0.4018'}
+step=  490  loss=  0.2571  grad_norm= 0.634  lr=7.42e-06  spk_acc=2047/2078
+
                                                      

 41%|████      | 490/1207 [4:43:52<6:44:34, 33.86s/it]
 41%|████      | 491/1207 [4:44:28<6:51:52, 34.52s/it]
 41%|████      | 492/1207 [4:44:59<6:39:52, 33.56s/it]
 41%|████      | 493/1207 [4:45:39<6:59:31, 35.25s/it]
 41%|████      | 494/1207 [4:46:04<6:23:19, 32.26s/it]
 41%|████      | 495/1207 [4:46:19<5:23:30, 27.26s/it]{'loss': '0.2571', 'grad_norm': '0.6337', 'learning_rate': '7.424e-06', 'epoch': '0.406'}
+step=  495  loss=  0.4766  grad_norm= 1.765  lr=7.36e-06  spk_acc=2675/2766
+
                                                      

 41%|████      | 495/1207 [4:46:19<5:23:30, 27.26s/it]
 41%|████      | 496/1207 [4:46:50<5:35:51, 28.34s/it]
 41%|████      | 497/1207 [4:47:26<6:00:39, 30.48s/it]
 41%|████▏     | 498/1207 [4:48:02<6:19:08, 32.09s/it]
 41%|████▏     | 499/1207 [4:48:35<6:21:22, 32.32s/it]
 41%|████▏     | 500/1207 [4:49:11<6:35:41, 33.58s/it]{'loss': '0.4766', 'grad_norm': '1.765', 'learning_rate': '7.361e-06', 'epoch': '0.4101'}
+step=  500  loss=  0.2753  grad_norm= 0.644  lr=7.30e-06  spk_acc=2466/2554
+
                                                      

 41%|████▏     | 500/1207 [4:49:11<6:35:41, 33.58s/it]
 42%|████▏     | 501/1207 [4:49:45<6:34:44, 33.55s/it]
 42%|████▏     | 502/1207 [4:50:16<6:27:09, 32.95s/it]
 42%|████▏     | 503/1207 [4:50:49<6:25:31, 32.86s/it]
 42%|████▏     | 504/1207 [4:51:28<6:49:09, 34.92s/it]
 42%|████▏     | 505/1207 [4:51:59<6:33:19, 33.62s/it]{'loss': '0.2753', 'grad_norm': '0.6439', 'learning_rate': '7.297e-06', 'epoch': '0.4143'}
+step=  505  loss=  0.2820  grad_norm= 1.113  lr=7.23e-06  spk_acc=3230/3301
+
                                                      

 42%|████▏     | 505/1207 [4:51:59<6:33:19, 33.62s/it]
 42%|████▏     | 506/1207 [4:52:32<6:30:54, 33.46s/it]
 42%|████▏     | 507/1207 [4:53:10<6:47:26, 34.92s/it]
 42%|████▏     | 508/1207 [4:53:58<7:31:39, 38.77s/it]
 42%|████▏     | 509/1207 [4:54:33<7:18:05, 37.66s/it]
 42%|████▏     | 510/1207 [4:55:11<7:16:41, 37.59s/it]{'loss': '0.282', 'grad_norm': '1.113', 'learning_rate': '7.232e-06', 'epoch': '0.4184'}
+step=  510  loss=  0.3161  grad_norm= 0.989  lr=7.17e-06  spk_acc=3679/3806
+
                                                      

 42%|████▏     | 510/1207 [4:55:11<7:16:41, 37.59s/it]
 42%|████▏     | 511/1207 [4:55:43<6:56:31, 35.91s/it]
 42%|████▏     | 512/1207 [4:56:16<6:48:10, 35.24s/it]
 43%|████▎     | 513/1207 [4:56:56<7:04:12, 36.68s/it]
 43%|████▎     | 514/1207 [4:57:30<6:51:51, 35.66s/it]
 43%|████▎     | 515/1207 [4:58:06<6:53:12, 35.83s/it]{'loss': '0.3161', 'grad_norm': '0.9886', 'learning_rate': '7.168e-06', 'epoch': '0.4225'}
+step=  515  loss=  0.2885  grad_norm= 0.507  lr=7.10e-06  spk_acc=2873/2977
+
                                                      

 43%|████▎     | 515/1207 [4:58:06<6:53:12, 35.83s/it]
 43%|████▎     | 516/1207 [4:58:39<6:44:00, 35.08s/it]
 43%|████▎     | 517/1207 [4:59:12<6:36:57, 34.52s/it]
 43%|████▎     | 518/1207 [4:59:51<6:49:59, 35.70s/it]
 43%|████▎     | 519/1207 [5:00:29<6:56:23, 36.31s/it]
 43%|████▎     | 520/1207 [5:01:05<6:54:37, 36.21s/it]{'loss': '0.2885', 'grad_norm': '0.5071', 'learning_rate': '7.102e-06', 'epoch': '0.4267'}
+step=  520  loss=  0.3293  grad_norm= 0.590  lr=7.04e-06  spk_acc=2764/2866
+
                                                      

 43%|████▎     | 520/1207 [5:01:05<6:54:37, 36.21s/it]
 43%|████▎     | 521/1207 [5:01:39<6:46:45, 35.58s/it]
 43%|████▎     | 522/1207 [5:02:11<6:35:22, 34.63s/it]
 43%|████▎     | 523/1207 [5:02:43<6:25:01, 33.77s/it]
 43%|████▎     | 524/1207 [5:03:16<6:21:26, 33.51s/it]
 43%|████▎     | 525/1207 [5:03:52<6:29:29, 34.27s/it]{'loss': '0.3293', 'grad_norm': '0.5903', 'learning_rate': '7.036e-06', 'epoch': '0.4308'}
+step=  525  loss=  0.2815  grad_norm= 0.546  lr=6.97e-06  spk_acc=2144/2194
+
                                                      

 43%|████▎     | 525/1207 [5:03:52<6:29:29, 34.27s/it]
 44%|████▎     | 526/1207 [5:04:29<6:39:01, 35.16s/it]
 44%|████▎     | 527/1207 [5:05:03<6:32:55, 34.67s/it]
 44%|████▎     | 528/1207 [5:05:41<6:46:24, 35.91s/it]
 44%|████▍     | 529/1207 [5:06:19<6:52:36, 36.51s/it]
 44%|████▍     | 530/1207 [5:06:54<6:45:49, 35.97s/it]{'loss': '0.2815', 'grad_norm': '0.5463', 'learning_rate': '6.97e-06', 'epoch': '0.435'}
+step=  530  loss=  0.2743  grad_norm= 0.490  lr=6.90e-06  spk_acc=2730/2787
+
                                                      

 44%|████▍     | 530/1207 [5:06:54<6:45:49, 35.97s/it]
 44%|████▍     | 531/1207 [5:07:11<5:39:27, 30.13s/it]
 44%|████▍     | 532/1207 [5:07:47<6:00:52, 32.08s/it]
 44%|████▍     | 533/1207 [5:08:20<6:01:30, 32.18s/it]
 44%|████▍     | 534/1207 [5:08:50<5:56:17, 31.77s/it]
 44%|████▍     | 535/1207 [5:09:28<6:15:07, 33.49s/it]{'loss': '0.2743', 'grad_norm': '0.4897', 'learning_rate': '6.903e-06', 'epoch': '0.4391'}
+step=  535  loss=  0.3347  grad_norm= 0.606  lr=6.84e-06  spk_acc=2166/2234
+
                                                      

 44%|████▍     | 535/1207 [5:09:28<6:15:07, 33.49s/it]
 44%|████▍     | 536/1207 [5:10:03<6:19:10, 33.90s/it]
 44%|████▍     | 537/1207 [5:10:41<6:32:37, 35.16s/it]
 45%|████▍     | 538/1207 [5:11:12<6:17:53, 33.89s/it]
 45%|████▍     | 539/1207 [5:11:45<6:15:13, 33.70s/it]
 45%|████▍     | 540/1207 [5:12:24<6:30:55, 35.17s/it]{'loss': '0.3347', 'grad_norm': '0.6058', 'learning_rate': '6.836e-06', 'epoch': '0.4432'}
+step=  540  loss=  0.3271  grad_norm= 0.458  lr=6.77e-06  spk_acc=2607/2716
+
                                                      

 45%|████▍     | 540/1207 [5:12:24<6:30:55, 35.17s/it]
 45%|████▍     | 541/1207 [5:13:04<6:46:48, 36.65s/it]
 45%|████▍     | 542/1207 [5:13:29<6:07:33, 33.16s/it]
 45%|████▍     | 543/1207 [5:14:04<6:12:21, 33.65s/it]
 45%|████▌     | 544/1207 [5:14:44<6:33:46, 35.64s/it]
 45%|████▌     | 545/1207 [5:15:19<6:32:47, 35.60s/it]{'loss': '0.3271', 'grad_norm': '0.4575', 'learning_rate': '6.769e-06', 'epoch': '0.4474'}
+step=  545  loss=  0.3828  grad_norm= 0.775  lr=6.70e-06  spk_acc=3010/3170
+
                                                      

 45%|████▌     | 545/1207 [5:15:19<6:32:47, 35.60s/it]
 45%|████▌     | 546/1207 [5:15:51<6:20:46, 34.56s/it]
 45%|████▌     | 547/1207 [5:16:29<6:29:39, 35.42s/it]
 45%|████▌     | 548/1207 [5:17:02<6:20:21, 34.63s/it]
 45%|████▌     | 549/1207 [5:17:38<6:26:35, 35.25s/it]
 46%|████▌     | 550/1207 [5:18:15<6:30:29, 35.66s/it]{'loss': '0.3828', 'grad_norm': '0.7748', 'learning_rate': '6.701e-06', 'epoch': '0.4515'}
+step=  550  loss=  0.3170  grad_norm= 0.589  lr=6.63e-06  spk_acc=2306/2371
+
                                                      

 46%|████▌     | 550/1207 [5:18:15<6:30:29, 35.66s/it]
 46%|████▌     | 551/1207 [5:18:49<6:22:54, 35.02s/it]
 46%|████▌     | 552/1207 [5:19:20<6:09:36, 33.86s/it]
 46%|████▌     | 553/1207 [5:19:53<6:06:22, 33.61s/it]
 46%|████▌     | 554/1207 [5:20:28<6:09:44, 33.97s/it]
 46%|████▌     | 555/1207 [5:21:04<6:17:48, 34.77s/it]{'loss': '0.317', 'grad_norm': '0.5894', 'learning_rate': '6.633e-06', 'epoch': '0.4557'}
+step=  555  loss=  0.3212  grad_norm= 0.543  lr=6.56e-06  spk_acc=2427/2490
+
                                                      

 46%|████▌     | 555/1207 [5:21:04<6:17:48, 34.77s/it]
 46%|████▌     | 556/1207 [5:21:39<6:17:26, 34.79s/it]
 46%|████▌     | 557/1207 [5:22:12<6:10:43, 34.22s/it]
 46%|████▌     | 558/1207 [5:22:40<5:49:40, 32.33s/it]
 46%|████▋     | 559/1207 [5:23:15<5:56:57, 33.05s/it]
 46%|████▋     | 560/1207 [5:23:50<6:05:28, 33.89s/it]{'loss': '0.3212', 'grad_norm': '0.5431', 'learning_rate': '6.564e-06', 'epoch': '0.4598'}
+step=  560  loss=  0.3115  grad_norm= 0.401  lr=6.50e-06  spk_acc=2149/2221
+
                                                      

 46%|████▋     | 560/1207 [5:23:50<6:05:28, 33.89s/it]
 46%|████▋     | 561/1207 [5:24:28<6:16:48, 35.00s/it]
 47%|████▋     | 562/1207 [5:25:06<6:24:59, 35.81s/it]
 47%|████▋     | 563/1207 [5:25:44<6:33:32, 36.67s/it]
 47%|████▋     | 564/1207 [5:26:19<6:26:56, 36.11s/it]
 47%|████▋     | 565/1207 [5:26:54<6:23:10, 35.81s/it]{'loss': '0.3115', 'grad_norm': '0.4012', 'learning_rate': '6.495e-06', 'epoch': '0.464'}
+step=  565  loss=  0.3561  grad_norm= 0.564  lr=6.43e-06  spk_acc=2894/3007
+
                                                      

 47%|████▋     | 565/1207 [5:26:54<6:23:10, 35.81s/it]
 47%|████▋     | 566/1207 [5:27:23<5:59:25, 33.64s/it]
 47%|████▋     | 567/1207 [5:27:58<6:04:19, 34.16s/it]
 47%|████▋     | 568/1207 [5:28:33<6:07:07, 34.47s/it]
 47%|████▋     | 569/1207 [5:29:07<6:03:00, 34.14s/it]
 47%|████▋     | 570/1207 [5:29:50<6:31:09, 36.84s/it]{'loss': '0.3561', 'grad_norm': '0.5643', 'learning_rate': '6.426e-06', 'epoch': '0.4681'}
+step=  570  loss=  0.3221  grad_norm= 0.547  lr=6.36e-06  spk_acc=3092/3219
+
                                                      

 47%|████▋     | 570/1207 [5:29:50<6:31:09, 36.84s/it]
 47%|████▋     | 571/1207 [5:30:22<6:17:01, 35.57s/it]
 47%|████▋     | 572/1207 [5:30:55<6:08:06, 34.78s/it]
 47%|████▋     | 573/1207 [5:31:25<5:50:25, 33.16s/it]
 48%|████▊     | 574/1207 [5:31:57<5:48:09, 33.00s/it]
 48%|████▊     | 575/1207 [5:32:32<5:52:24, 33.46s/it]{'loss': '0.3221', 'grad_norm': '0.5469', 'learning_rate': '6.357e-06', 'epoch': '0.4722'}
+step=  575  loss=  0.3264  grad_norm= 0.605  lr=6.29e-06  spk_acc=2049/2106
+
                                                      

 48%|████▊     | 575/1207 [5:32:32<5:52:24, 33.46s/it]
 48%|████▊     | 576/1207 [5:33:05<5:50:46, 33.35s/it]
 48%|████▊     | 577/1207 [5:33:41<5:59:51, 34.27s/it]
 48%|████▊     | 578/1207 [5:34:15<5:56:55, 34.05s/it]
 48%|████▊     | 579/1207 [5:34:46<5:48:04, 33.26s/it]
 48%|████▊     | 580/1207 [5:35:18<5:43:47, 32.90s/it]{'loss': '0.3264', 'grad_norm': '0.6053', 'learning_rate': '6.287e-06', 'epoch': '0.4764'}
+step=  580  loss=  0.3004  grad_norm= 0.753  lr=6.22e-06  spk_acc=2318/2403
+
                                                      

 48%|████▊     | 580/1207 [5:35:18<5:43:47, 32.90s/it]
 48%|████▊     | 581/1207 [5:35:51<5:42:37, 32.84s/it]
 48%|████▊     | 582/1207 [5:36:30<6:01:46, 34.73s/it]
 48%|████▊     | 583/1207 [5:37:08<6:11:34, 35.73s/it]
 48%|████▊     | 584/1207 [5:37:42<6:04:41, 35.12s/it]
 48%|████▊     | 585/1207 [5:38:12<5:48:58, 33.66s/it]{'loss': '0.3004', 'grad_norm': '0.7526', 'learning_rate': '6.217e-06', 'epoch': '0.4805'}
+step=  585  loss=  0.3025  grad_norm= 0.505  lr=6.15e-06  spk_acc=2638/2707
+
                                                      

 48%|████▊     | 585/1207 [5:38:12<5:48:58, 33.66s/it]
 49%|████▊     | 586/1207 [5:38:47<5:52:44, 34.08s/it]
 49%|████▊     | 587/1207 [5:39:20<5:48:27, 33.72s/it]
 49%|████▊     | 588/1207 [5:39:56<5:54:46, 34.39s/it]
 49%|████▉     | 589/1207 [5:40:44<6:35:42, 38.42s/it]
 49%|████▉     | 590/1207 [5:41:20<6:27:02, 37.64s/it]{'loss': '0.3025', 'grad_norm': '0.5053', 'learning_rate': '6.147e-06', 'epoch': '0.4847'}
+step=  590  loss=  0.3552  grad_norm= 0.371  lr=6.08e-06  spk_acc=3328/3515
+
                                                      

 49%|████▉     | 590/1207 [5:41:20<6:27:02, 37.64s/it]
 49%|████▉     | 591/1207 [5:41:54<6:16:13, 36.65s/it]
 49%|████▉     | 592/1207 [5:42:26<5:59:47, 35.10s/it]
 49%|████▉     | 593/1207 [5:42:57<5:47:28, 33.96s/it]
 49%|████▉     | 594/1207 [5:43:29<5:40:22, 33.32s/it]
 49%|████▉     | 595/1207 [5:44:03<5:41:39, 33.50s/it]{'loss': '0.3552', 'grad_norm': '0.3709', 'learning_rate': '6.076e-06', 'epoch': '0.4888'}
+step=  595  loss=  0.6431  grad_norm= 0.570  lr=6.01e-06  spk_acc=2148/2254
+
                                                      

 49%|████▉     | 595/1207 [5:44:03<5:41:39, 33.50s/it]
 49%|████▉     | 596/1207 [5:44:36<5:41:16, 33.51s/it]
 49%|████▉     | 597/1207 [5:45:13<5:50:56, 34.52s/it]
 50%|████▉     | 598/1207 [5:45:42<5:32:56, 32.80s/it]
 50%|████▉     | 599/1207 [5:46:17<5:38:15, 33.38s/it]
 50%|████▉     | 600/1207 [5:46:58<6:01:17, 35.71s/it]{'loss': '0.6431', 'grad_norm': '0.5704', 'learning_rate': '6.006e-06', 'epoch': '0.493'}
+step=  600  loss=  0.3167  grad_norm= 0.397  lr=5.93e-06  spk_acc=3153/3261
+
                                                      

 50%|████▉     | 600/1207 [5:46:58<6:01:17, 35.71s/it]
 50%|████▉     | 601/1207 [5:47:32<5:57:01, 35.35s/it]
 50%|████▉     | 602/1207 [5:48:06<5:51:57, 34.90s/it]
 50%|████▉     | 603/1207 [5:48:40<5:47:10, 34.49s/it]
 50%|█████     | 604/1207 [5:49:10<5:34:29, 33.28s/it]
 50%|█████     | 605/1207 [5:49:45<5:38:03, 33.69s/it]{'loss': '0.3167', 'grad_norm': '0.3969', 'learning_rate': '5.935e-06', 'epoch': '0.4971'}
+step=  605  loss=  0.2902  grad_norm= 0.571  lr=5.86e-06  spk_acc=2181/2245
+
                                                      

 50%|█████     | 605/1207 [5:49:45<5:38:03, 33.69s/it]
 50%|█████     | 606/1207 [5:50:19<5:38:06, 33.76s/it]
 50%|█████     | 607/1207 [5:50:55<5:45:56, 34.59s/it]
 50%|█████     | 608/1207 [5:51:27<5:37:27, 33.80s/it]
 50%|█████     | 609/1207 [5:52:00<5:32:19, 33.34s/it]
 51%|█████     | 610/1207 [5:52:31<5:27:03, 32.87s/it]{'loss': '0.2902', 'grad_norm': '0.5713', 'learning_rate': '5.863e-06', 'epoch': '0.5012'}
+step=  610  loss=  0.3313  grad_norm= 0.513  lr=5.79e-06  spk_acc=2624/2723
+
                                                      

 51%|█████     | 610/1207 [5:52:31<5:27:03, 32.87s/it]
 51%|█████     | 611/1207 [5:53:02<5:19:48, 32.19s/it]
 51%|█████     | 612/1207 [5:53:37<5:28:13, 33.10s/it]
 51%|█████     | 613/1207 [5:54:13<5:35:20, 33.87s/it]
 51%|█████     | 614/1207 [5:54:43<5:24:20, 32.82s/it]
 51%|█████     | 615/1207 [5:55:16<5:23:36, 32.80s/it]{'loss': '0.3313', 'grad_norm': '0.5128', 'learning_rate': '5.792e-06', 'epoch': '0.5054'}
+step=  615  loss=  0.2923  grad_norm= 0.578  lr=5.72e-06  spk_acc=2291/2345
+
                                                      

 51%|█████     | 615/1207 [5:55:16<5:23:36, 32.80s/it]
 51%|█████     | 616/1207 [5:55:47<5:18:39, 32.35s/it]
 51%|█████     | 617/1207 [5:56:21<5:22:31, 32.80s/it]
 51%|█████     | 618/1207 [5:56:58<5:33:13, 33.94s/it]
 51%|█████▏    | 619/1207 [5:57:28<5:23:08, 32.97s/it]
 51%|█████▏    | 620/1207 [5:57:59<5:15:36, 32.26s/it]{'loss': '0.2923', 'grad_norm': '0.5781', 'learning_rate': '5.721e-06', 'epoch': '0.5095'}
+step=  620  loss=  0.2708  grad_norm= 0.634  lr=5.65e-06  spk_acc=2234/2262
+
                                                      

 51%|█████▏    | 620/1207 [5:57:59<5:15:36, 32.26s/it]
 51%|█████▏    | 621/1207 [5:58:38<5:36:07, 34.42s/it]
 52%|█████▏    | 622/1207 [5:59:13<5:36:46, 34.54s/it]
 52%|█████▏    | 623/1207 [5:59:43<5:21:34, 33.04s/it]
 52%|█████▏    | 624/1207 [6:00:25<5:47:50, 35.80s/it]
 52%|█████▏    | 625/1207 [6:00:58<5:39:38, 35.01s/it]{'loss': '0.2708', 'grad_norm': '0.634', 'learning_rate': '5.649e-06', 'epoch': '0.5137'}
+step=  625  loss=  0.3201  grad_norm= 0.584  lr=5.58e-06  spk_acc=2676/2878
+
                                                      

 52%|█████▏    | 625/1207 [6:00:58<5:39:38, 35.01s/it]
 52%|█████▏    | 626/1207 [6:01:37<5:49:24, 36.08s/it]
 52%|█████▏    | 627/1207 [6:02:10<5:41:07, 35.29s/it]
 52%|█████▏    | 628/1207 [6:02:51<5:55:26, 36.83s/it]
 52%|█████▏    | 629/1207 [6:03:27<5:53:41, 36.71s/it]
 52%|█████▏    | 630/1207 [6:04:00<5:43:13, 35.69s/it]{'loss': '0.3201', 'grad_norm': '0.5845', 'learning_rate': '5.577e-06', 'epoch': '0.5178'}
+step=  630  loss=  0.3225  grad_norm= 0.645  lr=5.51e-06  spk_acc=2901/3052
+
                                                      

 52%|█████▏    | 630/1207 [6:04:00<5:43:13, 35.69s/it]
 52%|█████▏    | 631/1207 [6:04:40<5:52:36, 36.73s/it]
 52%|█████▏    | 632/1207 [6:05:12<5:39:40, 35.44s/it]
 52%|█████▏    | 633/1207 [6:05:48<5:39:55, 35.53s/it]
 53%|█████▎    | 634/1207 [6:06:25<5:43:32, 35.97s/it]
 53%|█████▎    | 635/1207 [6:06:57<5:31:47, 34.80s/it]{'loss': '0.3225', 'grad_norm': '0.6449', 'learning_rate': '5.505e-06', 'epoch': '0.522'}
+step=  635  loss=  0.3474  grad_norm= 0.476  lr=5.43e-06  spk_acc=2775/2873
+
                                                      

 53%|█████▎    | 635/1207 [6:06:57<5:31:47, 34.80s/it]
 53%|█████▎    | 636/1207 [6:07:36<5:44:23, 36.19s/it]
 53%|█████▎    | 637/1207 [6:08:10<5:38:04, 35.59s/it]
 53%|█████▎    | 638/1207 [6:08:41<5:22:27, 34.00s/it]
 53%|█████▎    | 639/1207 [6:09:18<5:31:07, 34.98s/it]
 53%|█████▎    | 640/1207 [6:09:55<5:36:32, 35.61s/it]{'loss': '0.3474', 'grad_norm': '0.4761', 'learning_rate': '5.433e-06', 'epoch': '0.5261'}
+step=  640  loss=  0.3422  grad_norm= 0.662  lr=5.36e-06  spk_acc=2515/2699
+
                                                      

 53%|█████▎    | 640/1207 [6:09:55<5:36:32, 35.61s/it]
 53%|█████▎    | 641/1207 [6:10:29<5:32:18, 35.23s/it]
 53%|█████▎    | 642/1207 [6:11:07<5:37:22, 35.83s/it]
 53%|█████▎    | 643/1207 [6:11:45<5:43:07, 36.50s/it]
 53%|█████▎    | 644/1207 [6:12:18<5:33:04, 35.50s/it]
 53%|█████▎    | 645/1207 [6:12:56<5:40:01, 36.30s/it]{'loss': '0.3422', 'grad_norm': '0.6619', 'learning_rate': '5.361e-06', 'epoch': '0.5302'}
+step=  645  loss=  0.3683  grad_norm= 0.605  lr=5.29e-06  spk_acc=2893/3016
+
                                                      

 53%|█████▎    | 645/1207 [6:12:56<5:40:01, 36.30s/it]
 54%|█████▎    | 646/1207 [6:13:38<5:54:46, 37.94s/it]
 54%|█████▎    | 647/1207 [6:14:10<5:36:41, 36.07s/it]
 54%|█████▎    | 648/1207 [6:14:46<5:38:22, 36.32s/it]
 54%|█████▍    | 649/1207 [6:15:25<5:43:38, 36.95s/it]
 54%|█████▍    | 650/1207 [6:15:58<5:31:51, 35.75s/it]{'loss': '0.3683', 'grad_norm': '0.6047', 'learning_rate': '5.289e-06', 'epoch': '0.5344'}
+step=  650  loss=  0.3450  grad_norm= 1.008  lr=5.22e-06  spk_acc=3426/3531
+
                                                      

 54%|█████▍    | 650/1207 [6:15:58<5:31:51, 35.75s/it]
 54%|█████▍    | 651/1207 [6:16:36<5:38:08, 36.49s/it]
 54%|█████▍    | 652/1207 [6:17:11<5:33:05, 36.01s/it]
 54%|█████▍    | 653/1207 [6:17:46<5:28:56, 35.63s/it]
 54%|█████▍    | 654/1207 [6:18:32<5:58:23, 38.89s/it]
 54%|█████▍    | 655/1207 [6:19:03<5:35:16, 36.44s/it]{'loss': '0.345', 'grad_norm': '1.008', 'learning_rate': '5.217e-06', 'epoch': '0.5385'}
+step=  655  loss=  0.3861  grad_norm= 0.591  lr=5.14e-06  spk_acc=3353/3480
+
                                                      

 54%|█████▍    | 655/1207 [6:19:03<5:35:16, 36.44s/it]
 54%|█████▍    | 656/1207 [6:19:38<5:31:36, 36.11s/it]
 54%|█████▍    | 657/1207 [6:20:14<5:30:52, 36.10s/it]
 55%|█████▍    | 658/1207 [6:20:43<5:09:53, 33.87s/it]
 55%|█████▍    | 659/1207 [6:21:21<5:22:09, 35.27s/it]
 55%|█████▍    | 660/1207 [6:21:54<5:13:44, 34.41s/it]{'loss': '0.3861', 'grad_norm': '0.5912', 'learning_rate': '5.145e-06', 'epoch': '0.5427'}
+step=  660  loss=  0.3793  grad_norm= 0.606  lr=5.07e-06  spk_acc=2686/2824
+
                                                      

 55%|█████▍    | 660/1207 [6:21:54<5:13:44, 34.41s/it]
 55%|█████▍    | 661/1207 [6:22:30<5:17:15, 34.86s/it]
 55%|█████▍    | 662/1207 [6:23:01<5:07:07, 33.81s/it]
 55%|█████▍    | 663/1207 [6:23:32<4:59:04, 32.99s/it]
 55%|█████▌    | 664/1207 [6:24:06<5:00:32, 33.21s/it]
 55%|█████▌    | 665/1207 [6:24:36<4:52:34, 32.39s/it]{'loss': '0.3793', 'grad_norm': '0.6063', 'learning_rate': '5.072e-06', 'epoch': '0.5468'}
+step=  665  loss=  0.2786  grad_norm= 0.439  lr=5.00e-06  spk_acc=1862/1876
+
                                                      

 55%|█████▌    | 665/1207 [6:24:36<4:52:34, 32.39s/it]
 55%|█████▌    | 666/1207 [6:25:16<5:10:37, 34.45s/it]
 55%|█████▌    | 667/1207 [6:25:57<5:29:26, 36.60s/it]
 55%|█████▌    | 668/1207 [6:26:29<5:16:57, 35.28s/it]
 55%|█████▌    | 669/1207 [6:27:00<5:02:41, 33.76s/it]
 56%|█████▌    | 670/1207 [6:27:23<4:33:32, 30.56s/it]{'loss': '0.2786', 'grad_norm': '0.4391', 'learning_rate': '5e-06', 'epoch': '0.551'}
+step=  670  loss=  0.3555  grad_norm= 0.606  lr=4.93e-06  spk_acc=2581/2669
+
                                                      

 56%|█████▌    | 670/1207 [6:27:23<4:33:32, 30.56s/it]
 56%|█████▌    | 671/1207 [6:27:59<4:47:56, 32.23s/it]
 56%|█████▌    | 672/1207 [6:28:33<4:53:07, 32.87s/it]
 56%|█████▌    | 673/1207 [6:29:12<5:08:33, 34.67s/it]
 56%|█████▌    | 674/1207 [6:29:46<5:05:29, 34.39s/it]
 56%|█████▌    | 675/1207 [6:30:20<5:05:16, 34.43s/it]{'loss': '0.3555', 'grad_norm': '0.6065', 'learning_rate': '4.928e-06', 'epoch': '0.5551'}
+step=  675  loss=  0.3028  grad_norm= 0.631  lr=4.86e-06  spk_acc=2513/2568
+
                                                      

 56%|█████▌    | 675/1207 [6:30:20<5:05:16, 34.43s/it]
 56%|█████▌    | 676/1207 [6:30:52<4:57:35, 33.63s/it]
 56%|█████▌    | 677/1207 [6:31:31<5:10:38, 35.17s/it]
 56%|█████▌    | 678/1207 [6:32:11<5:23:00, 36.64s/it]
 56%|█████▋    | 679/1207 [6:32:45<5:15:08, 35.81s/it]
 56%|█████▋    | 680/1207 [6:33:17<5:04:00, 34.61s/it]{'loss': '0.3028', 'grad_norm': '0.6314', 'learning_rate': '4.855e-06', 'epoch': '0.5592'}
+step=  680  loss=  0.3661  grad_norm= 0.671  lr=4.78e-06  spk_acc=2533/2617
+
                                                      

 56%|█████▋    | 680/1207 [6:33:17<5:04:00, 34.61s/it]
 56%|█████▋    | 681/1207 [6:33:47<4:51:46, 33.28s/it]
 57%|█████▋    | 682/1207 [6:34:24<5:00:08, 34.30s/it]
 57%|█████▋    | 683/1207 [6:34:54<4:49:55, 33.20s/it]
 57%|█████▋    | 684/1207 [6:35:26<4:46:42, 32.89s/it]
 57%|█████▋    | 685/1207 [6:36:00<4:47:06, 33.00s/it]{'loss': '0.3661', 'grad_norm': '0.671', 'learning_rate': '4.783e-06', 'epoch': '0.5634'}
+step=  685  loss=  0.3822  grad_norm= 0.446  lr=4.71e-06  spk_acc=2342/2441
+
                                                      

 57%|█████▋    | 685/1207 [6:36:00<4:47:06, 33.00s/it]
 57%|█████▋    | 686/1207 [6:36:34<4:51:19, 33.55s/it]
 57%|█████▋    | 687/1207 [6:37:13<5:03:36, 35.03s/it]
 57%|█████▋    | 688/1207 [6:37:50<5:08:34, 35.67s/it]
 57%|█████▋    | 689/1207 [6:38:25<5:06:24, 35.49s/it]
 57%|█████▋    | 690/1207 [6:38:58<4:58:38, 34.66s/it]{'loss': '0.3822', 'grad_norm': '0.4459', 'learning_rate': '4.711e-06', 'epoch': '0.5675'}
+step=  690  loss=  0.3565  grad_norm= 0.609  lr=4.64e-06  spk_acc=2926/3047
+
                                                      

 57%|█████▋    | 690/1207 [6:38:58<4:58:38, 34.66s/it]
 57%|█████▋    | 691/1207 [6:39:29<4:48:17, 33.52s/it]
 57%|█████▋    | 692/1207 [6:39:59<4:40:18, 32.66s/it]
 57%|█████▋    | 693/1207 [6:40:36<4:49:15, 33.77s/it]
 57%|█████▋    | 694/1207 [6:41:07<4:41:08, 32.88s/it]
 58%|█████▊    | 695/1207 [6:41:40<4:42:59, 33.16s/it]{'loss': '0.3565', 'grad_norm': '0.6088', 'learning_rate': '4.639e-06', 'epoch': '0.5717'}
+step=  695  loss=  0.3394  grad_norm= 0.507  lr=4.57e-06  spk_acc=2074/2129
+
                                                      

 58%|█████▊    | 695/1207 [6:41:40<4:42:59, 33.16s/it]
 58%|█████▊    | 696/1207 [6:42:12<4:37:44, 32.61s/it]
 58%|█████▊    | 697/1207 [6:42:47<4:43:30, 33.35s/it]
 58%|█████▊    | 698/1207 [6:43:16<4:32:34, 32.13s/it]
 58%|█████▊    | 699/1207 [6:43:55<4:49:28, 34.19s/it]
 58%|█████▊    | 700/1207 [6:44:33<4:57:24, 35.20s/it]{'loss': '0.3394', 'grad_norm': '0.5071', 'learning_rate': '4.567e-06', 'epoch': '0.5758'}
+step=  700  loss=  0.3677  grad_norm= 0.753  lr=4.49e-06  spk_acc=2695/2831
+
                                                      

 58%|█████▊    | 700/1207 [6:44:33<4:57:24, 35.20s/it]
 58%|█████▊    | 701/1207 [6:45:05<4:50:49, 34.49s/it]
 58%|█████▊    | 702/1207 [6:45:37<4:42:07, 33.52s/it]
 58%|█████▊    | 703/1207 [6:46:12<4:46:11, 34.07s/it]
 58%|█████▊    | 704/1207 [6:46:45<4:42:41, 33.72s/it]
 58%|█████▊    | 705/1207 [6:47:19<4:42:38, 33.78s/it]{'loss': '0.3677', 'grad_norm': '0.7531', 'learning_rate': '4.495e-06', 'epoch': '0.58'}
+step=  705  loss=  0.2849  grad_norm= 0.501  lr=4.42e-06  spk_acc=2663/2714
+
                                                      

 58%|█████▊    | 705/1207 [6:47:19<4:42:38, 33.78s/it]
 58%|█████▊    | 706/1207 [6:47:53<4:42:55, 33.88s/it]
 59%|█████▊    | 707/1207 [6:48:26<4:40:36, 33.67s/it]
 59%|█████▊    | 708/1207 [6:48:59<4:37:52, 33.41s/it]
 59%|█████▊    | 709/1207 [6:49:42<5:02:18, 36.42s/it]
 59%|█████▉    | 710/1207 [6:50:23<5:11:34, 37.61s/it]{'loss': '0.2849', 'grad_norm': '0.5007', 'learning_rate': '4.423e-06', 'epoch': '0.5841'}
+step=  710  loss=  0.2985  grad_norm= 0.424  lr=4.35e-06  spk_acc=3393/3495
+
                                                      

 59%|█████▉    | 710/1207 [6:50:23<5:11:34, 37.61s/it]
 59%|█████▉    | 711/1207 [6:50:56<5:01:01, 36.41s/it]
 59%|█████▉    | 712/1207 [6:51:33<5:01:00, 36.49s/it]
 59%|█████▉    | 713/1207 [6:52:11<5:04:06, 36.94s/it]
 59%|█████▉    | 714/1207 [6:52:40<4:42:47, 34.42s/it]
 59%|█████▉    | 715/1207 [6:53:20<4:57:07, 36.24s/it]{'loss': '0.2985', 'grad_norm': '0.4237', 'learning_rate': '4.351e-06', 'epoch': '0.5882'}
+step=  715  loss=  0.4002  grad_norm= 0.563  lr=4.28e-06  spk_acc=3178/3426
+
                                                      

 59%|█████▉    | 715/1207 [6:53:20<4:57:07, 36.24s/it]
 59%|█████▉    | 716/1207 [6:53:52<4:45:39, 34.91s/it]
 59%|█████▉    | 717/1207 [6:54:31<4:54:35, 36.07s/it]
 59%|█████▉    | 718/1207 [6:55:12<5:07:10, 37.69s/it]
 60%|█████▉    | 719/1207 [6:55:49<5:04:38, 37.46s/it]
 60%|█████▉    | 720/1207 [6:56:24<4:57:44, 36.68s/it]{'loss': '0.4002', 'grad_norm': '0.5631', 'learning_rate': '4.279e-06', 'epoch': '0.5924'}
+step=  720  loss=  0.3361  grad_norm= 0.842  lr=4.21e-06  spk_acc=2912/3050
+
                                                      

 60%|█████▉    | 720/1207 [6:56:24<4:57:44, 36.68s/it]
 60%|█████▉    | 721/1207 [6:57:03<5:03:17, 37.44s/it]
 60%|█████▉    | 722/1207 [6:57:36<4:50:23, 35.92s/it]
 60%|█████▉    | 723/1207 [6:58:19<5:07:28, 38.12s/it]
 60%|█████▉    | 724/1207 [6:58:52<4:55:01, 36.65s/it]
 60%|██████    | 725/1207 [6:59:27<4:50:06, 36.11s/it]{'loss': '0.3361', 'grad_norm': '0.8423', 'learning_rate': '4.208e-06', 'epoch': '0.5965'}
+step=  725  loss=  0.3161  grad_norm= 0.518  lr=4.14e-06  spk_acc=3056/3238
+
                                                      

 60%|██████    | 725/1207 [6:59:27<4:50:06, 36.11s/it]
 60%|██████    | 726/1207 [7:00:00<4:42:03, 35.18s/it]
 60%|██████    | 727/1207 [7:00:31<4:31:54, 33.99s/it]
 60%|██████    | 728/1207 [7:01:14<4:53:47, 36.80s/it]
 60%|██████    | 729/1207 [7:01:47<4:43:26, 35.58s/it]
 60%|██████    | 730/1207 [7:02:28<4:56:08, 37.25s/it]{'loss': '0.3161', 'grad_norm': '0.5179', 'learning_rate': '4.137e-06', 'epoch': '0.6007'}
+step=  730  loss=  0.3629  grad_norm= 0.785  lr=4.07e-06  spk_acc=3612/3804
+
                                                      

 60%|██████    | 730/1207 [7:02:28<4:56:08, 37.25s/it]
 61%|██████    | 731/1207 [7:03:05<4:54:57, 37.18s/it]
 61%|██████    | 732/1207 [7:03:43<4:54:31, 37.20s/it]
 61%|██████    | 733/1207 [7:04:17<4:46:28, 36.26s/it]
 61%|██████    | 734/1207 [7:04:50<4:39:55, 35.51s/it]
 61%|██████    | 735/1207 [7:05:34<4:57:25, 37.81s/it]{'loss': '0.3629', 'grad_norm': '0.7852', 'learning_rate': '4.065e-06', 'epoch': '0.6048'}
+step=  735  loss=  0.3758  grad_norm= 0.566  lr=3.99e-06  spk_acc=3138/3294
+
                                                      

 61%|██████    | 735/1207 [7:05:34<4:57:25, 37.81s/it]
 61%|██████    | 736/1207 [7:06:09<4:51:52, 37.18s/it]
 61%|██████    | 737/1207 [7:06:41<4:38:07, 35.51s/it]
 61%|██████    | 738/1207 [7:07:18<4:41:14, 35.98s/it]
 61%|██████    | 739/1207 [7:07:49<4:28:41, 34.45s/it]
 61%|██████▏   | 740/1207 [7:08:22<4:23:57, 33.91s/it]{'loss': '0.3758', 'grad_norm': '0.5664', 'learning_rate': '3.994e-06', 'epoch': '0.6089'}
+step=  740  loss=  0.2996  grad_norm= 0.625  lr=3.92e-06  spk_acc=2255/2324
+
                                                      

 61%|██████▏   | 740/1207 [7:08:22<4:23:57, 33.91s/it]
 61%|██████▏   | 741/1207 [7:08:56<4:24:54, 34.11s/it]
 61%|██████▏   | 742/1207 [7:09:31<4:26:19, 34.36s/it]
 62%|██████▏   | 743/1207 [7:10:11<4:38:36, 36.03s/it]
 62%|██████▏   | 744/1207 [7:10:44<4:30:22, 35.04s/it]
 62%|██████▏   | 745/1207 [7:11:17<4:26:18, 34.59s/it]{'loss': '0.2996', 'grad_norm': '0.6246', 'learning_rate': '3.924e-06', 'epoch': '0.6131'}
+step=  745  loss=  0.3706  grad_norm= 0.566  lr=3.85e-06  spk_acc=2637/2759
+
                                                      

 62%|██████▏   | 745/1207 [7:11:17<4:26:18, 34.59s/it]
 62%|██████▏   | 746/1207 [7:11:50<4:20:57, 33.96s/it]
 62%|██████▏   | 747/1207 [7:12:24<4:21:50, 34.15s/it]
 62%|██████▏   | 748/1207 [7:13:02<4:28:48, 35.14s/it]
 62%|██████▏   | 749/1207 [7:13:36<4:27:12, 35.00s/it]
 62%|██████▏   | 750/1207 [7:14:17<4:38:30, 36.57s/it]{'loss': '0.3706', 'grad_norm': '0.5657', 'learning_rate': '3.853e-06', 'epoch': '0.6172'}
+step=  750  loss=  0.3803  grad_norm= 0.781  lr=3.78e-06  spk_acc=2836/2956
+
                                                      

 62%|██████▏   | 750/1207 [7:14:17<4:38:30, 36.57s/it]
 62%|██████▏   | 751/1207 [7:14:54<4:39:18, 36.75s/it]
 62%|██████▏   | 752/1207 [7:15:27<4:30:27, 35.67s/it]
 62%|██████▏   | 753/1207 [7:15:58<4:20:25, 34.42s/it]
 62%|██████▏   | 754/1207 [7:16:40<4:35:03, 36.43s/it]
 63%|██████▎   | 755/1207 [7:17:16<4:33:19, 36.28s/it]{'loss': '0.3803', 'grad_norm': '0.7808', 'learning_rate': '3.783e-06', 'epoch': '0.6214'}
+step=  755  loss=  0.2973  grad_norm= 0.465  lr=3.71e-06  spk_acc=2932/3018
+
                                                      

 63%|██████▎   | 755/1207 [7:17:16<4:33:19, 36.28s/it]
 63%|██████▎   | 756/1207 [7:17:49<4:25:52, 35.37s/it]
 63%|██████▎   | 757/1207 [7:18:35<4:50:08, 38.68s/it]
 63%|██████▎   | 758/1207 [7:19:10<4:39:51, 37.40s/it]
 63%|██████▎   | 759/1207 [7:19:43<4:31:05, 36.31s/it]
 63%|██████▎   | 760/1207 [7:20:19<4:29:26, 36.17s/it]{'loss': '0.2973', 'grad_norm': '0.4647', 'learning_rate': '3.713e-06', 'epoch': '0.6255'}
+step=  760  loss=  0.3821  grad_norm= 0.655  lr=3.64e-06  spk_acc=2908/3057
+
                                                      

 63%|██████▎   | 760/1207 [7:20:19<4:29:26, 36.17s/it]
 63%|██████▎   | 761/1207 [7:20:54<4:25:07, 35.67s/it]
 63%|██████▎   | 762/1207 [7:21:26<4:15:53, 34.50s/it]
 63%|██████▎   | 763/1207 [7:22:04<4:24:27, 35.74s/it]
 63%|██████▎   | 764/1207 [7:22:46<4:37:42, 37.61s/it]
 63%|██████▎   | 765/1207 [7:23:29<4:47:42, 39.05s/it]{'loss': '0.3821', 'grad_norm': '0.655', 'learning_rate': '3.643e-06', 'epoch': '0.6297'}
+step=  765  loss=  0.4345  grad_norm= 0.795  lr=3.57e-06  spk_acc=3369/3627
+
                                                      

 63%|██████▎   | 765/1207 [7:23:29<4:47:42, 39.05s/it]
 63%|██████▎   | 766/1207 [7:24:02<4:35:47, 37.52s/it]
 64%|██████▎   | 767/1207 [7:24:39<4:33:48, 37.34s/it]
 64%|██████▎   | 768/1207 [7:25:17<4:33:25, 37.37s/it]
 64%|██████▎   | 769/1207 [7:25:52<4:27:26, 36.63s/it]
 64%|██████▍   | 770/1207 [7:26:26<4:21:26, 35.90s/it]{'loss': '0.4345', 'grad_norm': '0.7952', 'learning_rate': '3.574e-06', 'epoch': '0.6338'}
+step=  770  loss=  0.2885  grad_norm= 0.354  lr=3.50e-06  spk_acc=3400/3452
+
                                                      

 64%|██████▍   | 770/1207 [7:26:26<4:21:26, 35.90s/it]
 64%|██████▍   | 771/1207 [7:27:00<4:17:09, 35.39s/it]
 64%|██████▍   | 772/1207 [7:27:47<4:42:02, 38.90s/it]
 64%|██████▍   | 773/1207 [7:28:22<4:32:17, 37.64s/it]
 64%|██████▍   | 774/1207 [7:28:57<4:26:42, 36.96s/it]
 64%|██████▍   | 775/1207 [7:29:32<4:21:44, 36.35s/it]{'loss': '0.2885', 'grad_norm': '0.3545', 'learning_rate': '3.505e-06', 'epoch': '0.6379'}
+step=  775  loss=  0.3693  grad_norm= 0.566  lr=3.44e-06  spk_acc=2993/3072
+
                                                      

 64%|██████▍   | 775/1207 [7:29:32<4:21:44, 36.35s/it]
 64%|██████▍   | 776/1207 [7:30:08<4:20:01, 36.20s/it]
 64%|██████▍   | 777/1207 [7:30:39<4:09:06, 34.76s/it]
 64%|██████▍   | 778/1207 [7:31:08<3:56:11, 33.03s/it]
 65%|██████▍   | 779/1207 [7:31:42<3:57:33, 33.30s/it]
 65%|██████▍   | 780/1207 [7:32:18<4:02:26, 34.07s/it]{'loss': '0.3693', 'grad_norm': '0.5665', 'learning_rate': '3.436e-06', 'epoch': '0.6421'}
+step=  780  loss=  0.3795  grad_norm= 0.467  lr=3.37e-06  spk_acc=3083/3193
+
                                                      

 65%|██████▍   | 780/1207 [7:32:18<4:02:26, 34.07s/it]
 65%|██████▍   | 781/1207 [7:32:44<3:45:03, 31.70s/it]
 65%|██████▍   | 782/1207 [7:33:21<3:54:27, 33.10s/it]
 65%|██████▍   | 783/1207 [7:33:48<3:40:55, 31.26s/it]
 65%|██████▍   | 784/1207 [7:33:56<2:51:06, 24.27s/it]
 65%|██████▌   | 785/1207 [7:34:28<3:08:07, 26.75s/it]{'loss': '0.3795', 'grad_norm': '0.4665', 'learning_rate': '3.367e-06', 'epoch': '0.6462'}
+step=  785  loss=  0.6945  grad_norm= 0.596  lr=3.30e-06  spk_acc=2127/2265
+
                                                      

 65%|██████▌   | 785/1207 [7:34:28<3:08:07, 26.75s/it]
 65%|██████▌   | 786/1207 [7:34:55<3:08:24, 26.85s/it]
 65%|██████▌   | 787/1207 [7:35:31<3:27:09, 29.59s/it]
 65%|██████▌   | 788/1207 [7:35:59<3:23:11, 29.10s/it]
 65%|██████▌   | 789/1207 [7:36:35<3:36:45, 31.11s/it]
 65%|██████▌   | 790/1207 [7:37:08<3:39:52, 31.64s/it]{'loss': '0.6945', 'grad_norm': '0.596', 'learning_rate': '3.299e-06', 'epoch': '0.6504'}
+step=  790  loss=  0.3552  grad_norm= 0.500  lr=3.23e-06  spk_acc=2412/2471
+
                                                      

 65%|██████▌   | 790/1207 [7:37:08<3:39:52, 31.64s/it]
 66%|██████▌   | 791/1207 [7:37:37<3:32:56, 30.71s/it]
 66%|██████▌   | 792/1207 [7:38:13<3:43:53, 32.37s/it]
 66%|██████▌   | 793/1207 [7:38:51<3:55:22, 34.11s/it]
 66%|██████▌   | 794/1207 [7:39:25<3:54:45, 34.10s/it]
 66%|██████▌   | 795/1207 [7:40:01<3:57:24, 34.57s/it]{'loss': '0.3552', 'grad_norm': '0.5001', 'learning_rate': '3.231e-06', 'epoch': '0.6545'}
+step=  795  loss=  0.3555  grad_norm= 0.365  lr=3.16e-06  spk_acc=2594/2691
+
                                                      

 66%|██████▌   | 795/1207 [7:40:01<3:57:24, 34.57s/it]
 66%|██████▌   | 796/1207 [7:40:31<3:48:36, 33.37s/it]
 66%|██████▌   | 797/1207 [7:41:11<4:01:56, 35.41s/it]
 66%|██████▌   | 798/1207 [7:41:45<3:58:36, 35.00s/it]
 66%|██████▌   | 799/1207 [7:42:18<3:52:42, 34.22s/it]
 66%|██████▋   | 800/1207 [7:42:48<3:43:16, 32.91s/it]{'loss': '0.3555', 'grad_norm': '0.3646', 'learning_rate': '3.164e-06', 'epoch': '0.6587'}
+step=  800  loss=  0.4235  grad_norm= 0.579  lr=3.10e-06  spk_acc=2857/3019
+
                                                      

 66%|██████▋   | 800/1207 [7:42:48<3:43:16, 32.91s/it]{'loss': '0.4235', 'grad_norm': '0.5789', 'learning_rate': '3.097e-06', 'epoch': '0.6628'}
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.02s/it]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.02s/it]
+/wangshuai/moss/MOSS_Speaker-RoPE/moss_speaker_rope/wespeaker_redimnet/tfmel.py:296: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+
 66%|██████▋   | 801/1207 [7:43:49<4:39:15, 41.27s/it]
 66%|██████▋   | 802/1207 [7:44:23<4:24:19, 39.16s/it]
 67%|██████▋   | 803/1207 [7:44:59<4:17:05, 38.18s/it]
 67%|██████▋   | 804/1207 [7:45:32<4:07:42, 36.88s/it]
 67%|██████▋   | 805/1207 [7:46:08<4:03:57, 36.41s/it]step=  805  loss=  0.3454  grad_norm= 0.648  lr=3.03e-06  spk_acc=2607/2648
+
                                                      

 67%|██████▋   | 805/1207 [7:46:08<4:03:57, 36.41s/it]
 67%|██████▋   | 806/1207 [7:46:46<4:05:56, 36.80s/it]
 67%|██████▋   | 807/1207 [7:47:22<4:05:42, 36.86s/it]
 67%|██████▋   | 808/1207 [7:47:57<4:01:03, 36.25s/it]
 67%|██████▋   | 809/1207 [7:48:28<3:49:02, 34.53s/it]
 67%|██████▋   | 810/1207 [7:48:44<3:12:48, 29.14s/it]{'loss': '0.3454', 'grad_norm': '0.6481', 'learning_rate': '3.03e-06', 'epoch': '0.6669'}
+step=  810  loss=  0.5299  grad_norm= 1.138  lr=2.96e-06  spk_acc=2770/2994
+
                                                      

 67%|██████▋   | 810/1207 [7:48:44<3:12:48, 29.14s/it]
 67%|██████▋   | 811/1207 [7:49:21<3:26:17, 31.26s/it]
 67%|██████▋   | 812/1207 [7:49:53<3:28:00, 31.60s/it]
 67%|██████▋   | 813/1207 [7:50:22<3:22:06, 30.78s/it]
 67%|██████▋   | 814/1207 [7:50:53<3:21:34, 30.78s/it]
 68%|██████▊   | 815/1207 [7:51:25<3:23:46, 31.19s/it]{'loss': '0.5299', 'grad_norm': '1.138', 'learning_rate': '2.964e-06', 'epoch': '0.6711'}
+step=  815  loss=  0.3366  grad_norm= 0.609  lr=2.90e-06  spk_acc=2410/2520
+
                                                      

 68%|██████▊   | 815/1207 [7:51:25<3:23:46, 31.19s/it]
 68%|██████▊   | 816/1207 [7:51:57<3:25:14, 31.50s/it]
 68%|██████▊   | 817/1207 [7:52:39<3:44:31, 34.54s/it]
 68%|██████▊   | 818/1207 [7:53:15<3:47:44, 35.13s/it]
 68%|██████▊   | 819/1207 [7:53:49<3:44:00, 34.64s/it]
 68%|██████▊   | 820/1207 [7:54:21<3:39:25, 34.02s/it]{'loss': '0.3366', 'grad_norm': '0.6089', 'learning_rate': '2.898e-06', 'epoch': '0.6752'}
+step=  820  loss=  0.3881  grad_norm= 0.756  lr=2.83e-06  spk_acc=2792/2931
+
                                                      

 68%|██████▊   | 820/1207 [7:54:21<3:39:25, 34.02s/it]
 68%|██████▊   | 821/1207 [7:54:44<3:16:34, 30.56s/it]
 68%|██████▊   | 822/1207 [7:55:17<3:21:53, 31.46s/it]
 68%|██████▊   | 823/1207 [7:55:51<3:26:12, 32.22s/it]
 68%|██████▊   | 824/1207 [7:56:26<3:30:41, 33.01s/it]
 68%|██████▊   | 825/1207 [7:57:06<3:42:44, 34.99s/it]{'loss': '0.3881', 'grad_norm': '0.7562', 'learning_rate': '2.832e-06', 'epoch': '0.6794'}
+step=  825  loss=  0.3246  grad_norm= 0.781  lr=2.77e-06  spk_acc=2872/2927
+
                                                      

 68%|██████▊   | 825/1207 [7:57:06<3:42:44, 34.99s/it]
 68%|██████▊   | 826/1207 [7:57:48<3:55:42, 37.12s/it]
 69%|██████▊   | 827/1207 [7:58:22<3:49:43, 36.27s/it]
 69%|██████▊   | 828/1207 [7:58:56<3:43:49, 35.43s/it]
 69%|██████▊   | 829/1207 [7:59:34<3:49:27, 36.42s/it]
 69%|██████▉   | 830/1207 [8:00:10<3:48:19, 36.34s/it]{'loss': '0.3246', 'grad_norm': '0.7809', 'learning_rate': '2.768e-06', 'epoch': '0.6835'}
+step=  830  loss=  0.4133  grad_norm= 0.814  lr=2.70e-06  spk_acc=3297/3469
+
                                                      

 69%|██████▉   | 830/1207 [8:00:10<3:48:19, 36.34s/it]
 69%|██████▉   | 831/1207 [8:00:49<3:52:43, 37.14s/it]
 69%|██████▉   | 832/1207 [8:01:24<3:47:33, 36.41s/it]
 69%|██████▉   | 833/1207 [8:02:04<3:52:29, 37.30s/it]
 69%|██████▉   | 834/1207 [8:02:41<3:51:16, 37.20s/it]
 69%|██████▉   | 835/1207 [8:03:27<4:08:13, 40.04s/it]{'loss': '0.4133', 'grad_norm': '0.8139', 'learning_rate': '2.703e-06', 'epoch': '0.6877'}
+step=  835  loss=  0.4071  grad_norm= 0.927  lr=2.64e-06  spk_acc=3860/4169
+
                                                      

 69%|██████▉   | 835/1207 [8:03:27<4:08:13, 40.04s/it]
 69%|██████▉   | 836/1207 [8:04:04<4:02:06, 39.15s/it]
 69%|██████▉   | 837/1207 [8:04:36<3:47:14, 36.85s/it]
 69%|██████▉   | 838/1207 [8:05:10<3:41:06, 35.95s/it]
 70%|██████▉   | 839/1207 [8:05:52<3:51:38, 37.77s/it]
 70%|██████▉   | 840/1207 [8:06:24<3:40:52, 36.11s/it]{'loss': '0.4071', 'grad_norm': '0.9274', 'learning_rate': '2.639e-06', 'epoch': '0.6918'}
+step=  840  loss=  0.4032  grad_norm= 0.596  lr=2.58e-06  spk_acc=2604/2916
+
                                                      

 70%|██████▉   | 840/1207 [8:06:24<3:40:52, 36.11s/it]
 70%|██████▉   | 841/1207 [8:06:59<3:37:43, 35.69s/it]
 70%|██████▉   | 842/1207 [8:07:31<3:31:20, 34.74s/it]
 70%|██████▉   | 843/1207 [8:08:06<3:31:24, 34.85s/it]
 70%|██████▉   | 844/1207 [8:08:51<3:49:44, 37.97s/it]
 70%|███████   | 845/1207 [8:09:24<3:39:18, 36.35s/it]{'loss': '0.4032', 'grad_norm': '0.596', 'learning_rate': '2.576e-06', 'epoch': '0.6959'}
+step=  845  loss=  0.3037  grad_norm= 0.642  lr=2.51e-06  spk_acc=3011/3061
+
                                                      

 70%|███████   | 845/1207 [8:09:24<3:39:18, 36.35s/it]
 70%|███████   | 846/1207 [8:10:10<3:56:49, 39.36s/it]
 70%|███████   | 847/1207 [8:10:48<3:52:12, 38.70s/it]
 70%|███████   | 848/1207 [8:11:23<3:46:02, 37.78s/it]
 70%|███████   | 849/1207 [8:11:59<3:41:17, 37.09s/it]
 70%|███████   | 850/1207 [8:12:25<3:21:14, 33.82s/it]{'loss': '0.3037', 'grad_norm': '0.642', 'learning_rate': '2.513e-06', 'epoch': '0.7001'}
+step=  850  loss=  0.3118  grad_norm= 0.667  lr=2.45e-06  spk_acc=2467/2577
+
                                                      

 70%|███████   | 850/1207 [8:12:25<3:21:14, 33.82s/it]
 71%|███████   | 851/1207 [8:13:01<3:25:14, 34.59s/it]
 71%|███████   | 852/1207 [8:13:40<3:31:33, 35.76s/it]
 71%|███████   | 853/1207 [8:14:20<3:38:57, 37.11s/it]
 71%|███████   | 854/1207 [8:14:52<3:29:08, 35.55s/it]
 71%|███████   | 855/1207 [8:15:30<3:33:17, 36.36s/it]{'loss': '0.3118', 'grad_norm': '0.6674', 'learning_rate': '2.45e-06', 'epoch': '0.7042'}
+step=  855  loss=  0.3810  grad_norm= 0.657  lr=2.39e-06  spk_acc=3322/3441
+
                                                      

 71%|███████   | 855/1207 [8:15:30<3:33:17, 36.36s/it]
 71%|███████   | 856/1207 [8:16:03<3:26:15, 35.26s/it]
 71%|███████   | 857/1207 [8:16:38<3:26:05, 35.33s/it]
 71%|███████   | 858/1207 [8:17:15<3:27:56, 35.75s/it]
 71%|███████   | 859/1207 [8:17:42<3:11:12, 32.97s/it]
 71%|███████▏  | 860/1207 [8:18:15<3:10:49, 33.00s/it]{'loss': '0.381', 'grad_norm': '0.6573', 'learning_rate': '2.388e-06', 'epoch': '0.7084'}
+step=  860  loss=  0.4064  grad_norm= 0.463  lr=2.33e-06  spk_acc=2643/2770
+
                                                      

 71%|███████▏  | 860/1207 [8:18:15<3:10:49, 33.00s/it]
 71%|███████▏  | 861/1207 [8:18:54<3:20:39, 34.80s/it]
 71%|███████▏  | 862/1207 [8:19:27<3:18:28, 34.52s/it]
 71%|███████▏  | 863/1207 [8:20:00<3:14:39, 33.95s/it]
 72%|███████▏  | 864/1207 [8:20:31<3:08:48, 33.03s/it]
 72%|███████▏  | 865/1207 [8:21:03<3:06:20, 32.69s/it]{'loss': '0.4064', 'grad_norm': '0.4629', 'learning_rate': '2.327e-06', 'epoch': '0.7125'}
+step=  865  loss=  0.3642  grad_norm= 1.105  lr=2.27e-06  spk_acc=2343/2505
+
                                                      

 72%|███████▏  | 865/1207 [8:21:03<3:06:20, 32.69s/it]
 72%|███████▏  | 866/1207 [8:21:41<3:14:49, 34.28s/it]
 72%|███████▏  | 867/1207 [8:22:19<3:20:08, 35.32s/it]
 72%|███████▏  | 868/1207 [8:22:58<3:26:44, 36.59s/it]
 72%|███████▏  | 869/1207 [8:23:35<3:27:17, 36.80s/it]
 72%|███████▏  | 870/1207 [8:24:12<3:25:27, 36.58s/it]{'loss': '0.3642', 'grad_norm': '1.105', 'learning_rate': '2.266e-06', 'epoch': '0.7167'}
+step=  870  loss=  0.3502  grad_norm= 0.610  lr=2.21e-06  spk_acc=3498/3675
+
                                                      

 72%|███████▏  | 870/1207 [8:24:12<3:25:27, 36.58s/it]
 72%|███████▏  | 871/1207 [8:24:46<3:20:47, 35.85s/it]
 72%|███████▏  | 872/1207 [8:25:14<3:08:23, 33.74s/it]
 72%|███████▏  | 873/1207 [8:25:57<3:21:59, 36.29s/it]
 72%|███████▏  | 874/1207 [8:26:27<3:12:07, 34.62s/it]
 72%|███████▏  | 875/1207 [8:27:00<3:07:32, 33.89s/it]{'loss': '0.3502', 'grad_norm': '0.6099', 'learning_rate': '2.206e-06', 'epoch': '0.7208'}
+step=  875  loss=  0.3650  grad_norm= 0.580  lr=2.15e-06  spk_acc=2269/2374
+
                                                      

 72%|███████▏  | 875/1207 [8:27:00<3:07:32, 33.89s/it]
 73%|███████▎  | 876/1207 [8:27:37<3:12:20, 34.87s/it]
 73%|███████▎  | 877/1207 [8:28:15<3:16:55, 35.80s/it]
 73%|███████▎  | 878/1207 [8:28:48<3:11:37, 34.95s/it]
 73%|███████▎  | 879/1207 [8:29:18<3:04:03, 33.67s/it]
 73%|███████▎  | 880/1207 [8:29:53<3:04:50, 33.91s/it]{'loss': '0.365', 'grad_norm': '0.5803', 'learning_rate': '2.146e-06', 'epoch': '0.7249'}
+step=  880  loss=  0.3805  grad_norm= 0.777  lr=2.09e-06  spk_acc=2311/2420
+
                                                      

 73%|███████▎  | 880/1207 [8:29:53<3:04:50, 33.91s/it]
 73%|███████▎  | 881/1207 [8:30:27<3:03:59, 33.86s/it]
 73%|███████▎  | 882/1207 [8:31:00<3:03:23, 33.86s/it]
 73%|███████▎  | 883/1207 [8:31:36<3:05:49, 34.41s/it]
 73%|███████▎  | 884/1207 [8:32:11<3:06:18, 34.61s/it]
 73%|███████▎  | 885/1207 [8:32:46<3:06:25, 34.74s/it]{'loss': '0.3805', 'grad_norm': '0.7771', 'learning_rate': '2.087e-06', 'epoch': '0.7291'}
+step=  885  loss=  0.3115  grad_norm= 0.433  lr=2.03e-06  spk_acc=2724/2819
+
                                                      

 73%|███████▎  | 885/1207 [8:32:46<3:06:25, 34.74s/it]
 73%|███████▎  | 886/1207 [8:33:20<3:04:38, 34.51s/it]
 73%|███████▎  | 887/1207 [8:33:52<2:59:11, 33.60s/it]
 74%|███████▎  | 888/1207 [8:34:27<3:00:45, 34.00s/it]
 74%|███████▎  | 889/1207 [8:34:59<2:58:17, 33.64s/it]
 74%|███████▎  | 890/1207 [8:35:38<3:05:57, 35.20s/it]{'loss': '0.3115', 'grad_norm': '0.4331', 'learning_rate': '2.028e-06', 'epoch': '0.7332'}
+step=  890  loss=  0.4129  grad_norm= 0.516  lr=1.97e-06  spk_acc=2700/2875
+
                                                      

 74%|███████▎  | 890/1207 [8:35:38<3:05:57, 35.20s/it]
 74%|███████▍  | 891/1207 [8:36:24<3:22:09, 38.38s/it]
 74%|███████▍  | 892/1207 [8:36:55<3:09:30, 36.10s/it]
 74%|███████▍  | 893/1207 [8:37:27<3:02:22, 34.85s/it]
 74%|███████▍  | 894/1207 [8:38:03<3:04:19, 35.33s/it]
 74%|███████▍  | 895/1207 [8:38:46<3:15:24, 37.58s/it]{'loss': '0.4129', 'grad_norm': '0.5161', 'learning_rate': '1.971e-06', 'epoch': '0.7374'}
+step=  895  loss=  0.4502  grad_norm= 1.089  lr=1.91e-06  spk_acc=3724/3957
+
                                                      

 74%|███████▍  | 895/1207 [8:38:46<3:15:24, 37.58s/it]
 74%|███████▍  | 896/1207 [8:39:27<3:20:11, 38.62s/it]
 74%|███████▍  | 897/1207 [8:40:01<3:11:51, 37.13s/it]
 74%|███████▍  | 898/1207 [8:40:39<3:12:54, 37.46s/it]
 74%|███████▍  | 899/1207 [8:41:07<2:56:55, 34.46s/it]
 75%|███████▍  | 900/1207 [8:41:40<2:55:29, 34.30s/it]{'loss': '0.4502', 'grad_norm': '1.089', 'learning_rate': '1.913e-06', 'epoch': '0.7415'}
+step=  900  loss=  0.3855  grad_norm= 0.623  lr=1.86e-06  spk_acc=2955/3089
+
                                                      

 75%|███████▍  | 900/1207 [8:41:40<2:55:29, 34.30s/it]
 75%|███████▍  | 901/1207 [8:42:17<2:59:02, 35.10s/it]
 75%|███████▍  | 902/1207 [8:42:49<2:52:47, 33.99s/it]
 75%|███████▍  | 903/1207 [8:43:22<2:51:33, 33.86s/it]
 75%|███████▍  | 904/1207 [8:43:55<2:48:42, 33.41s/it]
 75%|███████▍  | 905/1207 [8:44:26<2:44:45, 32.73s/it]{'loss': '0.3855', 'grad_norm': '0.6228', 'learning_rate': '1.857e-06', 'epoch': '0.7457'}
+step=  905  loss=  0.4220  grad_norm= 0.890  lr=1.80e-06  spk_acc=2583/2744
+
                                                      

 75%|███████▍  | 905/1207 [8:44:26<2:44:45, 32.73s/it]
 75%|███████▌  | 906/1207 [8:44:57<2:41:47, 32.25s/it]
 75%|███████▌  | 907/1207 [8:45:34<2:47:58, 33.59s/it]
 75%|███████▌  | 908/1207 [8:46:12<2:54:55, 35.10s/it]
 75%|███████▌  | 909/1207 [8:46:43<2:47:43, 33.77s/it]
 75%|███████▌  | 910/1207 [8:47:13<2:41:33, 32.64s/it]{'loss': '0.422', 'grad_norm': '0.8897', 'learning_rate': '1.801e-06', 'epoch': '0.7498'}
+step=  910  loss=  0.3910  grad_norm= 0.449  lr=1.75e-06  spk_acc=3296/3458
+
                                                      

 75%|███████▌  | 910/1207 [8:47:13<2:41:33, 32.64s/it]
 75%|███████▌  | 911/1207 [8:47:47<2:43:06, 33.06s/it]
 76%|███████▌  | 912/1207 [8:48:18<2:38:56, 32.33s/it]
 76%|███████▌  | 913/1207 [8:48:50<2:38:31, 32.35s/it]
 76%|███████▌  | 914/1207 [8:49:25<2:41:59, 33.17s/it]
 76%|███████▌  | 915/1207 [8:49:57<2:39:55, 32.86s/it]{'loss': '0.391', 'grad_norm': '0.4486', 'learning_rate': '1.746e-06', 'epoch': '0.7539'}
+step=  915  loss=  0.3387  grad_norm= 0.553  lr=1.69e-06  spk_acc=2046/2083
+
                                                      

 76%|███████▌  | 915/1207 [8:49:57<2:39:55, 32.86s/it]
 76%|███████▌  | 916/1207 [8:50:28<2:36:47, 32.33s/it]
 76%|███████▌  | 917/1207 [8:50:59<2:34:13, 31.91s/it]
 76%|███████▌  | 918/1207 [8:51:33<2:36:22, 32.46s/it]
 76%|███████▌  | 919/1207 [8:52:12<2:45:25, 34.46s/it]
 76%|███████▌  | 920/1207 [8:52:49<2:48:37, 35.25s/it]{'loss': '0.3387', 'grad_norm': '0.5534', 'learning_rate': '1.691e-06', 'epoch': '0.7581'}
+step=  920  loss=  0.3563  grad_norm= 0.667  lr=1.64e-06  spk_acc=2999/3138
+
                                                      

 76%|███████▌  | 920/1207 [8:52:49<2:48:37, 35.25s/it]
 76%|███████▋  | 921/1207 [8:53:25<2:49:09, 35.49s/it]
 76%|███████▋  | 922/1207 [8:54:00<2:48:03, 35.38s/it]
 76%|███████▋  | 923/1207 [8:54:38<2:51:00, 36.13s/it]
 77%|███████▋  | 924/1207 [8:55:10<2:44:17, 34.83s/it]
 77%|███████▋  | 925/1207 [8:55:43<2:41:04, 34.27s/it]{'loss': '0.3563', 'grad_norm': '0.6669', 'learning_rate': '1.637e-06', 'epoch': '0.7622'}
+step=  925  loss=  0.3688  grad_norm= 0.665  lr=1.58e-06  spk_acc=2537/2648
+
                                                      

 77%|███████▋  | 925/1207 [8:55:43<2:41:04, 34.27s/it]
 77%|███████▋  | 926/1207 [8:56:00<2:15:47, 29.00s/it]
 77%|███████▋  | 927/1207 [8:56:33<2:21:48, 30.39s/it]
 77%|███████▋  | 928/1207 [8:57:08<2:27:26, 31.71s/it]
 77%|███████▋  | 929/1207 [8:57:45<2:33:51, 33.21s/it]
 77%|███████▋  | 930/1207 [8:58:18<2:32:44, 33.09s/it]{'loss': '0.3688', 'grad_norm': '0.6647', 'learning_rate': '1.584e-06', 'epoch': '0.7664'}
+step=  930  loss=  0.4456  grad_norm= 0.995  lr=1.53e-06  spk_acc=2261/2434
+
                                                      

 77%|███████▋  | 930/1207 [8:58:18<2:32:44, 33.09s/it]
 77%|███████▋  | 931/1207 [8:58:55<2:38:17, 34.41s/it]
 77%|███████▋  | 932/1207 [8:59:25<2:31:09, 32.98s/it]
 77%|███████▋  | 933/1207 [8:59:59<2:32:26, 33.38s/it]
 77%|███████▋  | 934/1207 [9:00:35<2:34:53, 34.04s/it]
 77%|███████▋  | 935/1207 [9:01:08<2:33:15, 33.81s/it]{'loss': '0.4456', 'grad_norm': '0.9954', 'learning_rate': '1.532e-06', 'epoch': '0.7705'}
+step=  935  loss=  0.3331  grad_norm= 0.665  lr=1.48e-06  spk_acc=2663/2730
+
                                                      

 77%|███████▋  | 935/1207 [9:01:08<2:33:15, 33.81s/it]
 78%|███████▊  | 936/1207 [9:01:45<2:37:27, 34.86s/it]
 78%|███████▊  | 937/1207 [9:02:18<2:33:15, 34.06s/it]
 78%|███████▊  | 938/1207 [9:02:53<2:34:04, 34.36s/it]
 78%|███████▊  | 939/1207 [9:03:27<2:34:09, 34.51s/it]
 78%|███████▊  | 940/1207 [9:03:57<2:27:22, 33.12s/it]{'loss': '0.3331', 'grad_norm': '0.6651', 'learning_rate': '1.48e-06', 'epoch': '0.7746'}
+step=  940  loss=  0.4017  grad_norm= 0.882  lr=1.43e-06  spk_acc=3056/3190
+
                                                      

 78%|███████▊  | 940/1207 [9:03:57<2:27:22, 33.12s/it]
 78%|███████▊  | 941/1207 [9:04:35<2:32:51, 34.48s/it]
 78%|███████▊  | 942/1207 [9:05:10<2:33:18, 34.71s/it]
 78%|███████▊  | 943/1207 [9:05:43<2:30:29, 34.20s/it]
 78%|███████▊  | 944/1207 [9:06:14<2:25:20, 33.16s/it]
 78%|███████▊  | 945/1207 [9:06:46<2:23:00, 32.75s/it]{'loss': '0.4017', 'grad_norm': '0.8816', 'learning_rate': '1.429e-06', 'epoch': '0.7788'}
+step=  945  loss=  0.3567  grad_norm= 0.669  lr=1.38e-06  spk_acc=2765/2973
+
                                                      

 78%|███████▊  | 945/1207 [9:06:46<2:23:00, 32.75s/it]
 78%|███████▊  | 946/1207 [9:06:57<1:54:14, 26.26s/it]
 78%|███████▊  | 947/1207 [9:07:27<1:59:19, 27.53s/it]
 79%|███████▊  | 948/1207 [9:08:00<2:05:38, 29.11s/it]
 79%|███████▊  | 949/1207 [9:08:33<2:09:36, 30.14s/it]
 79%|███████▊  | 950/1207 [9:09:08<2:15:04, 31.54s/it]{'loss': '0.3567', 'grad_norm': '0.6688', 'learning_rate': '1.379e-06', 'epoch': '0.7829'}
+step=  950  loss=  0.6473  grad_norm= 2.105  lr=1.33e-06  spk_acc=2222/2328
+
                                                      

 79%|███████▊  | 950/1207 [9:09:08<2:15:04, 31.54s/it]
 79%|███████▉  | 951/1207 [9:09:45<2:21:40, 33.20s/it]
 79%|███████▉  | 952/1207 [9:10:17<2:20:30, 33.06s/it]
 79%|███████▉  | 953/1207 [9:10:55<2:25:48, 34.44s/it]
 79%|███████▉  | 954/1207 [9:11:26<2:20:59, 33.44s/it]
 79%|███████▉  | 955/1207 [9:11:54<2:13:26, 31.77s/it]{'loss': '0.6473', 'grad_norm': '2.105', 'learning_rate': '1.329e-06', 'epoch': '0.7871'}
+step=  955  loss=  0.3749  grad_norm= 0.931  lr=1.28e-06  spk_acc=2860/2984
+
                                                      

 79%|███████▉  | 955/1207 [9:11:54<2:13:26, 31.77s/it]
 79%|███████▉  | 956/1207 [9:12:27<2:14:34, 32.17s/it]
 79%|███████▉  | 957/1207 [9:12:58<2:13:06, 31.94s/it]
 79%|███████▉  | 958/1207 [9:13:10<1:47:28, 25.90s/it]
 79%|███████▉  | 959/1207 [9:13:40<1:51:25, 26.96s/it]
 80%|███████▉  | 960/1207 [9:14:15<2:00:45, 29.33s/it]{'loss': '0.3749', 'grad_norm': '0.9308', 'learning_rate': '1.28e-06', 'epoch': '0.7912'}
+step=  960  loss=  0.7338  grad_norm= 0.707  lr=1.23e-06  spk_acc=2308/2552
+
                                                      

 80%|███████▉  | 960/1207 [9:14:15<2:00:45, 29.33s/it]
 80%|███████▉  | 961/1207 [9:14:47<2:04:02, 30.25s/it]
 80%|███████▉  | 962/1207 [9:15:17<2:03:42, 30.29s/it]
 80%|███████▉  | 963/1207 [9:15:59<2:16:26, 33.55s/it]
 80%|███████▉  | 964/1207 [9:16:35<2:19:06, 34.35s/it]
 80%|███████▉  | 965/1207 [9:17:05<2:12:59, 32.97s/it]{'loss': '0.7338', 'grad_norm': '0.7066', 'learning_rate': '1.232e-06', 'epoch': '0.7954'}
+step=  965  loss=  0.3782  grad_norm= 0.709  lr=1.19e-06  spk_acc=2349/2453
+
                                                      

 80%|███████▉  | 965/1207 [9:17:05<2:12:59, 32.97s/it]
 80%|████████  | 966/1207 [9:17:39<2:14:29, 33.48s/it]
 80%|████████  | 967/1207 [9:18:29<2:33:15, 38.32s/it]
 80%|████████  | 968/1207 [9:19:00<2:23:48, 36.10s/it]
 80%|████████  | 969/1207 [9:19:36<2:23:40, 36.22s/it]
 80%|████████  | 970/1207 [9:20:09<2:19:09, 35.23s/it]{'loss': '0.3782', 'grad_norm': '0.7088', 'learning_rate': '1.185e-06', 'epoch': '0.7995'}
+step=  970  loss=  0.4164  grad_norm= 0.755  lr=1.14e-06  spk_acc=3312/3546
+
                                                      

 80%|████████  | 970/1207 [9:20:09<2:19:09, 35.23s/it]
 80%|████████  | 971/1207 [9:20:34<2:06:41, 32.21s/it]
 81%|████████  | 972/1207 [9:21:08<2:08:07, 32.71s/it]
 81%|████████  | 973/1207 [9:21:41<2:07:12, 32.62s/it]
 81%|████████  | 974/1207 [9:22:15<2:08:27, 33.08s/it]
 81%|████████  | 975/1207 [9:22:52<2:12:24, 34.24s/it]{'loss': '0.4164', 'grad_norm': '0.7552', 'learning_rate': '1.139e-06', 'epoch': '0.8036'}
+step=  975  loss=  0.3918  grad_norm= 0.524  lr=1.09e-06  spk_acc=2666/2772
+
                                                      

 81%|████████  | 975/1207 [9:22:52<2:12:24, 34.24s/it]
 81%|████████  | 976/1207 [9:23:27<2:13:10, 34.59s/it]
 81%|████████  | 977/1207 [9:23:58<2:08:32, 33.53s/it]
 81%|████████  | 978/1207 [9:24:28<2:03:52, 32.46s/it]
 81%|████████  | 979/1207 [9:25:00<2:03:11, 32.42s/it]
 81%|████████  | 980/1207 [9:25:37<2:06:57, 33.56s/it]{'loss': '0.3918', 'grad_norm': '0.5237', 'learning_rate': '1.093e-06', 'epoch': '0.8078'}
+step=  980  loss=  0.3792  grad_norm= 0.529  lr=1.05e-06  spk_acc=2610/2712
+
                                                      

 81%|████████  | 980/1207 [9:25:37<2:06:57, 33.56s/it]
 81%|████████▏ | 981/1207 [9:26:12<2:08:45, 34.18s/it]
 81%|████████▏ | 982/1207 [9:26:51<2:13:02, 35.48s/it]
 81%|████████▏ | 983/1207 [9:27:34<2:21:25, 37.88s/it]
 82%|████████▏ | 984/1207 [9:28:06<2:13:39, 35.96s/it]
 82%|████████▏ | 985/1207 [9:28:44<2:15:41, 36.67s/it]{'loss': '0.3792', 'grad_norm': '0.5292', 'learning_rate': '1.049e-06', 'epoch': '0.8119'}
+step=  985  loss=  0.4349  grad_norm= 0.611  lr=1.00e-06  spk_acc=3121/3291
+
                                                      

 82%|████████▏ | 985/1207 [9:28:44<2:15:41, 36.67s/it]
 82%|████████▏ | 986/1207 [9:29:20<2:14:07, 36.42s/it]
 82%|████████▏ | 987/1207 [9:29:51<2:07:38, 34.81s/it]
 82%|████████▏ | 988/1207 [9:30:28<2:09:22, 35.45s/it]
 82%|████████▏ | 989/1207 [9:30:59<2:04:27, 34.25s/it]
 82%|████████▏ | 990/1207 [9:31:32<2:01:59, 33.73s/it]{'loss': '0.4349', 'grad_norm': '0.6111', 'learning_rate': '1.005e-06', 'epoch': '0.8161'}
+step=  990  loss=  0.3831  grad_norm= 0.638  lr=9.62e-07  spk_acc=2307/2411
+
                                                      

 82%|████████▏ | 990/1207 [9:31:32<2:01:59, 33.73s/it]
 82%|████████▏ | 991/1207 [9:32:02<1:57:53, 32.75s/it]
 82%|████████▏ | 992/1207 [9:32:37<1:59:27, 33.34s/it]
 82%|████████▏ | 993/1207 [9:33:03<1:50:54, 31.09s/it]
 82%|████████▏ | 994/1207 [9:33:34<1:49:57, 30.97s/it]
 82%|████████▏ | 995/1207 [9:34:08<1:52:37, 31.88s/it]{'loss': '0.3831', 'grad_norm': '0.6383', 'learning_rate': '9.617e-07', 'epoch': '0.8202'}
+step=  995  loss=  0.4139  grad_norm= 0.661  lr=9.20e-07  spk_acc=1775/1867
+
                                                      

 82%|████████▏ | 995/1207 [9:34:08<1:52:37, 31.88s/it]
 83%|████████▎ | 996/1207 [9:34:42<1:54:41, 32.61s/it]
 83%|████████▎ | 997/1207 [9:35:18<1:57:42, 33.63s/it]
 83%|████████▎ | 998/1207 [9:35:50<1:55:42, 33.22s/it]
 83%|████████▎ | 999/1207 [9:36:25<1:57:18, 33.84s/it]
 83%|████████▎ | 1000/1207 [9:37:03<2:01:03, 35.09s/it]{'loss': '0.4139', 'grad_norm': '0.6607', 'learning_rate': '9.195e-07', 'epoch': '0.8244'}
+step= 1000  loss=  0.3883  grad_norm= 0.905  lr=8.78e-07  spk_acc=2976/3185
+
                                                       

 83%|████████▎ | 1000/1207 [9:37:03<2:01:03, 35.09s/it]
 83%|████████▎ | 1001/1207 [9:37:42<2:03:47, 36.06s/it]
 83%|████████▎ | 1002/1207 [9:38:18<2:03:19, 36.09s/it]
 83%|████████▎ | 1003/1207 [9:38:48<1:56:15, 34.20s/it]
 83%|████████▎ | 1004/1207 [9:39:23<1:56:27, 34.42s/it]
 83%|████████▎ | 1005/1207 [9:39:53<1:51:51, 33.23s/it]{'loss': '0.3883', 'grad_norm': '0.9053', 'learning_rate': '8.781e-07', 'epoch': '0.8285'}
+step= 1005  loss=  0.4037  grad_norm= 1.046  lr=8.38e-07  spk_acc=2190/2379
+
                                                       

 83%|████████▎ | 1005/1207 [9:39:53<1:51:51, 33.23s/it]
 83%|████████▎ | 1006/1207 [9:40:26<1:51:14, 33.21s/it]
 83%|████████▎ | 1007/1207 [9:40:56<1:46:56, 32.08s/it]
 84%|████████▎ | 1008/1207 [9:41:32<1:50:35, 33.34s/it]
 84%|████████▎ | 1009/1207 [9:42:05<1:49:46, 33.26s/it]
 84%|████████▎ | 1010/1207 [9:42:34<1:45:07, 32.02s/it]{'loss': '0.4037', 'grad_norm': '1.046', 'learning_rate': '8.376e-07', 'epoch': '0.8326'}
+step= 1010  loss=  0.4419  grad_norm= 0.665  lr=7.98e-07  spk_acc=3215/3384
+
                                                       

 84%|████████▎ | 1010/1207 [9:42:34<1:45:07, 32.02s/it]
 84%|████████▍ | 1011/1207 [9:43:12<1:49:50, 33.63s/it]
 84%|████████▍ | 1012/1207 [9:43:45<1:49:06, 33.57s/it]
 84%|████████▍ | 1013/1207 [9:44:18<1:47:50, 33.35s/it]
 84%|████████▍ | 1014/1207 [9:44:51<1:47:09, 33.31s/it]
 84%|████████▍ | 1015/1207 [9:45:27<1:49:07, 34.10s/it]{'loss': '0.4419', 'grad_norm': '0.6647', 'learning_rate': '7.98e-07', 'epoch': '0.8368'}
+step= 1015  loss=  0.3176  grad_norm= 0.722  lr=7.59e-07  spk_acc=2529/2598
+
                                                       

 84%|████████▍ | 1015/1207 [9:45:27<1:49:07, 34.10s/it]
 84%|████████▍ | 1016/1207 [9:46:02<1:49:35, 34.42s/it]
 84%|████████▍ | 1017/1207 [9:46:42<1:53:51, 35.95s/it]
 84%|████████▍ | 1018/1207 [9:47:12<1:48:09, 34.34s/it]
 84%|████████▍ | 1019/1207 [9:47:47<1:47:54, 34.44s/it]
 85%|████████▍ | 1020/1207 [9:48:18<1:43:54, 33.34s/it]{'loss': '0.3176', 'grad_norm': '0.7219', 'learning_rate': '7.592e-07', 'epoch': '0.8409'}
+step= 1020  loss=  0.4842  grad_norm= 1.125  lr=7.21e-07  spk_acc=2758/2968
+
                                                       

 85%|████████▍ | 1020/1207 [9:48:18<1:43:54, 33.34s/it]
 85%|████████▍ | 1021/1207 [9:48:44<1:36:36, 31.16s/it]
 85%|████████▍ | 1022/1207 [9:49:17<1:38:10, 31.84s/it]
 85%|████████▍ | 1023/1207 [9:49:51<1:39:23, 32.41s/it]
 85%|████████▍ | 1024/1207 [9:50:31<1:46:05, 34.78s/it]
 85%|████████▍ | 1025/1207 [9:51:11<1:50:08, 36.31s/it]{'loss': '0.4842', 'grad_norm': '1.125', 'learning_rate': '7.214e-07', 'epoch': '0.8451'}
+step= 1025  loss=  0.3838  grad_norm= 0.564  lr=6.84e-07  spk_acc=3010/3193
+
                                                       

 85%|████████▍ | 1025/1207 [9:51:11<1:50:08, 36.31s/it]
 85%|████████▌ | 1026/1207 [9:51:45<1:47:33, 35.66s/it]
 85%|████████▌ | 1027/1207 [9:52:16<1:42:12, 34.07s/it]
 85%|████████▌ | 1028/1207 [9:52:52<1:43:32, 34.71s/it]
 85%|████████▌ | 1029/1207 [9:53:23<1:40:02, 33.72s/it]
 85%|████████▌ | 1030/1207 [9:53:53<1:35:45, 32.46s/it]{'loss': '0.3838', 'grad_norm': '0.5645', 'learning_rate': '6.844e-07', 'epoch': '0.8492'}
+step= 1030  loss=  0.4502  grad_norm= 0.809  lr=6.48e-07  spk_acc=1897/1992
+
                                                       

 85%|████████▌ | 1030/1207 [9:53:53<1:35:45, 32.46s/it]
 85%|████████▌ | 1031/1207 [9:54:26<1:35:24, 32.53s/it]
 86%|████████▌ | 1032/1207 [9:54:56<1:33:30, 32.06s/it]
 86%|████████▌ | 1033/1207 [9:55:31<1:35:31, 32.94s/it]
 86%|████████▌ | 1034/1207 [9:56:03<1:33:52, 32.56s/it]
 86%|████████▌ | 1035/1207 [9:56:36<1:33:49, 32.73s/it]{'loss': '0.4502', 'grad_norm': '0.8091', 'learning_rate': '6.483e-07', 'epoch': '0.8534'}
+step= 1035  loss=  0.3743  grad_norm= 0.424  lr=6.13e-07  spk_acc=2091/2191
+
                                                       

 86%|████████▌ | 1035/1207 [9:56:36<1:33:49, 32.73s/it]
 86%|████████▌ | 1036/1207 [9:57:10<1:34:05, 33.01s/it]
 86%|████████▌ | 1037/1207 [9:57:46<1:36:04, 33.91s/it]
 86%|████████▌ | 1038/1207 [9:58:19<1:34:24, 33.52s/it]
 86%|████████▌ | 1039/1207 [9:58:51<1:32:46, 33.13s/it]
 86%|████████▌ | 1040/1207 [9:59:26<1:33:43, 33.67s/it]{'loss': '0.3743', 'grad_norm': '0.4238', 'learning_rate': '6.132e-07', 'epoch': '0.8575'}
+step= 1040  loss=  0.4313  grad_norm= 0.501  lr=5.79e-07  spk_acc=2443/2597
+
                                                       

 86%|████████▌ | 1040/1207 [9:59:26<1:33:43, 33.67s/it]
 86%|████████▌ | 1041/1207 [9:59:58<1:31:35, 33.11s/it]
 86%|████████▋ | 1042/1207 [10:00:31<1:31:39, 33.33s/it]
 86%|████████▋ | 1043/1207 [10:00:59<1:26:40, 31.71s/it]
 86%|████████▋ | 1044/1207 [10:01:34<1:28:18, 32.50s/it]
 87%|████████▋ | 1045/1207 [10:02:06<1:27:54, 32.56s/it]{'loss': '0.4313', 'grad_norm': '0.5011', 'learning_rate': '5.789e-07', 'epoch': '0.8616'}
+step= 1045  loss=  0.4340  grad_norm= 0.968  lr=5.46e-07  spk_acc=2644/2836
+
                                                        

 87%|████████▋ | 1045/1207 [10:02:06<1:27:54, 32.56s/it]
 87%|████████▋ | 1046/1207 [10:02:42<1:29:33, 33.37s/it]
 87%|████████▋ | 1047/1207 [10:03:17<1:30:42, 34.02s/it]
 87%|████████▋ | 1048/1207 [10:04:07<1:42:44, 38.77s/it]
 87%|████████▋ | 1049/1207 [10:04:44<1:40:30, 38.17s/it]
 87%|████████▋ | 1050/1207 [10:05:19<1:37:17, 37.18s/it]{'loss': '0.434', 'grad_norm': '0.9677', 'learning_rate': '5.456e-07', 'epoch': '0.8658'}
+step= 1050  loss=  0.4150  grad_norm= 0.638  lr=5.13e-07  spk_acc=3659/3896
+
                                                        

 87%|████████▋ | 1050/1207 [10:05:19<1:37:17, 37.18s/it]
 87%|████████▋ | 1051/1207 [10:05:54<1:34:57, 36.52s/it]
 87%|████████▋ | 1052/1207 [10:06:30<1:34:00, 36.39s/it]
 87%|████████▋ | 1053/1207 [10:07:08<1:35:08, 37.07s/it]
 87%|████████▋ | 1054/1207 [10:07:41<1:31:23, 35.84s/it]
 87%|████████▋ | 1055/1207 [10:08:20<1:33:07, 36.76s/it]{'loss': '0.415', 'grad_norm': '0.6385', 'learning_rate': '5.132e-07', 'epoch': '0.8699'}
+step= 1055  loss=  0.3940  grad_norm= 0.438  lr=4.82e-07  spk_acc=3377/3558
+
                                                        

 87%|████████▋ | 1055/1207 [10:08:20<1:33:07, 36.76s/it]
 87%|████████▋ | 1056/1207 [10:08:59<1:33:42, 37.23s/it]
 88%|████████▊ | 1057/1207 [10:09:32<1:30:22, 36.15s/it]
 88%|████████▊ | 1058/1207 [10:10:07<1:29:07, 35.89s/it]
 88%|████████▊ | 1059/1207 [10:10:39<1:25:35, 34.70s/it]
 88%|████████▊ | 1060/1207 [10:11:13<1:23:52, 34.23s/it]{'loss': '0.394', 'grad_norm': '0.4384', 'learning_rate': '4.818e-07', 'epoch': '0.8741'}
+step= 1060  loss=  0.4382  grad_norm= 0.791  lr=4.51e-07  spk_acc=2560/2737
+
                                                        

 88%|████████▊ | 1060/1207 [10:11:13<1:23:52, 34.23s/it]
 88%|████████▊ | 1061/1207 [10:11:51<1:26:01, 35.36s/it]
 88%|████████▊ | 1062/1207 [10:12:26<1:25:10, 35.25s/it]
 88%|████████▊ | 1063/1207 [10:13:00<1:23:54, 34.96s/it]
 88%|████████▊ | 1064/1207 [10:13:36<1:24:21, 35.39s/it]
 88%|████████▊ | 1065/1207 [10:14:09<1:21:48, 34.56s/it]{'loss': '0.4382', 'grad_norm': '0.7906', 'learning_rate': '4.513e-07', 'epoch': '0.8782'}
+step= 1065  loss=  0.4233  grad_norm= 0.848  lr=4.22e-07  spk_acc=2676/2828
+
                                                        

 88%|████████▊ | 1065/1207 [10:14:09<1:21:48, 34.56s/it]
 88%|████████▊ | 1066/1207 [10:14:43<1:20:53, 34.42s/it]
 88%|████████▊ | 1067/1207 [10:15:22<1:23:16, 35.69s/it]
 88%|████████▊ | 1068/1207 [10:16:06<1:28:59, 38.41s/it]
 89%|████████▊ | 1069/1207 [10:16:42<1:26:18, 37.52s/it]
 89%|████████▊ | 1070/1207 [10:17:12<1:20:36, 35.30s/it]{'loss': '0.4233', 'grad_norm': '0.8479', 'learning_rate': '4.217e-07', 'epoch': '0.8824'}
+step= 1070  loss=  0.4355  grad_norm= 0.448  lr=3.93e-07  spk_acc=3313/3568
+
                                                        

 89%|████████▊ | 1070/1207 [10:17:12<1:20:36, 35.30s/it]
 89%|████████▊ | 1071/1207 [10:17:46<1:19:10, 34.93s/it]
 89%|████████▉ | 1072/1207 [10:18:17<1:15:50, 33.71s/it]
 89%|████████▉ | 1073/1207 [10:18:53<1:16:59, 34.47s/it]
 89%|████████▉ | 1074/1207 [10:19:29<1:17:25, 34.93s/it]
 89%|████████▉ | 1075/1207 [10:20:03<1:16:17, 34.68s/it]{'loss': '0.4355', 'grad_norm': '0.4483', 'learning_rate': '3.932e-07', 'epoch': '0.8865'}
+step= 1075  loss=  0.4107  grad_norm= 0.471  lr=3.66e-07  spk_acc=2244/2346
+
                                                        

 89%|████████▉ | 1075/1207 [10:20:03<1:16:17, 34.68s/it]
 89%|████████▉ | 1076/1207 [10:20:40<1:17:18, 35.41s/it]
 89%|████████▉ | 1077/1207 [10:21:14<1:15:41, 34.94s/it]
 89%|████████▉ | 1078/1207 [10:21:52<1:17:05, 35.85s/it]
 89%|████████▉ | 1079/1207 [10:22:28<1:16:25, 35.83s/it]
 89%|████████▉ | 1080/1207 [10:23:03<1:15:06, 35.48s/it]{'loss': '0.4107', 'grad_norm': '0.4706', 'learning_rate': '3.655e-07', 'epoch': '0.8906'}
+step= 1080  loss=  0.4677  grad_norm= 0.666  lr=3.39e-07  spk_acc=2535/2746
+
                                                        

 89%|████████▉ | 1080/1207 [10:23:03<1:15:06, 35.48s/it]
 90%|████████▉ | 1081/1207 [10:23:38<1:14:25, 35.44s/it]
 90%|████████▉ | 1082/1207 [10:24:12<1:12:47, 34.94s/it]
 90%|████████▉ | 1083/1207 [10:24:44<1:10:19, 34.03s/it]
 90%|████████▉ | 1084/1207 [10:25:14<1:07:37, 32.99s/it]
 90%|████████▉ | 1085/1207 [10:25:48<1:07:45, 33.32s/it]{'loss': '0.4677', 'grad_norm': '0.6658', 'learning_rate': '3.389e-07', 'epoch': '0.8948'}
+step= 1085  loss=  0.3558  grad_norm= 0.616  lr=3.13e-07  spk_acc=2261/2328
+
                                                        

 90%|████████▉ | 1085/1207 [10:25:48<1:07:45, 33.32s/it]
 90%|████████▉ | 1086/1207 [10:26:22<1:07:44, 33.59s/it]
 90%|█████████ | 1087/1207 [10:26:57<1:07:35, 33.79s/it]
 90%|█████████ | 1088/1207 [10:27:14<57:21, 28.92s/it]  
 90%|█████████ | 1089/1207 [10:27:47<59:00, 30.01s/it]
 90%|█████████ | 1090/1207 [10:28:19<59:36, 30.57s/it]{'loss': '0.3558', 'grad_norm': '0.616', 'learning_rate': '3.132e-07', 'epoch': '0.8989'}
+step= 1090  loss=  0.4617  grad_norm= 0.847  lr=2.88e-07  spk_acc=2118/2216
+
                                                      

 90%|█████████ | 1090/1207 [10:28:19<59:36, 30.57s/it]
 90%|█████████ | 1091/1207 [10:28:52<1:00:39, 31.38s/it]
 90%|█████████ | 1092/1207 [10:29:26<1:01:55, 32.31s/it]
 91%|█████████ | 1093/1207 [10:30:01<1:02:52, 33.09s/it]
 91%|█████████ | 1094/1207 [10:30:33<1:01:15, 32.52s/it]
 91%|█████████ | 1095/1207 [10:31:01<58:09, 31.15s/it]  {'loss': '0.4617', 'grad_norm': '0.8467', 'learning_rate': '2.885e-07', 'epoch': '0.9031'}
+step= 1095  loss=  0.3583  grad_norm= 0.807  lr=2.65e-07  spk_acc=3168/3269
+
                                                      

 91%|█████████ | 1095/1207 [10:31:01<58:09, 31.15s/it]
 91%|█████████ | 1096/1207 [10:31:39<1:01:54, 33.46s/it]
 91%|█████████ | 1097/1207 [10:32:14<1:02:08, 33.90s/it]
 91%|█████████ | 1098/1207 [10:32:41<57:54, 31.88s/it]  
 91%|█████████ | 1099/1207 [10:33:13<57:15, 31.81s/it]
 91%|█████████ | 1100/1207 [10:33:50<59:14, 33.22s/it]{'loss': '0.3583', 'grad_norm': '0.8067', 'learning_rate': '2.648e-07', 'epoch': '0.9072'}
+step= 1100  loss=  0.3920  grad_norm= 0.509  lr=2.42e-07  spk_acc=2401/2523
+
                                                      

 91%|█████████ | 1100/1207 [10:33:50<59:14, 33.22s/it]
 91%|█████████ | 1101/1207 [10:34:19<56:41, 32.09s/it]
 91%|█████████▏| 1102/1207 [10:34:50<55:38, 31.80s/it]
 91%|█████████▏| 1103/1207 [10:35:22<54:53, 31.67s/it]
 91%|█████████▏| 1104/1207 [10:35:57<56:16, 32.78s/it]
 92%|█████████▏| 1105/1207 [10:36:31<56:21, 33.15s/it]{'loss': '0.392', 'grad_norm': '0.5093', 'learning_rate': '2.42e-07', 'epoch': '0.9114'}
+step= 1105  loss=  0.4337  grad_norm= 0.783  lr=2.20e-07  spk_acc=2163/2232
+
                                                      

 92%|█████████▏| 1105/1207 [10:36:31<56:21, 33.15s/it]
 92%|█████████▏| 1106/1207 [10:37:06<56:35, 33.62s/it]
 92%|█████████▏| 1107/1207 [10:37:41<56:54, 34.14s/it]
 92%|█████████▏| 1108/1207 [10:38:17<57:19, 34.75s/it]
 92%|█████████▏| 1109/1207 [10:38:51<56:26, 34.55s/it]
 92%|█████████▏| 1110/1207 [10:39:36<1:00:48, 37.61s/it]{'loss': '0.4337', 'grad_norm': '0.7829', 'learning_rate': '2.203e-07', 'epoch': '0.9155'}
+step= 1110  loss=  0.3825  grad_norm= 0.991  lr=2.00e-07  spk_acc=2929/3037
+
                                                        

 92%|█████████▏| 1110/1207 [10:39:36<1:00:48, 37.61s/it]
 92%|█████████▏| 1111/1207 [10:40:07<56:52, 35.55s/it]  
 92%|█████████▏| 1112/1207 [10:40:40<55:17, 34.92s/it]
 92%|█████████▏| 1113/1207 [10:41:14<54:06, 34.54s/it]
 92%|█████████▏| 1114/1207 [10:41:50<54:28, 35.15s/it]
 92%|█████████▏| 1115/1207 [10:42:26<53:57, 35.19s/it]{'loss': '0.3825', 'grad_norm': '0.9911', 'learning_rate': '1.996e-07', 'epoch': '0.9196'}
+step= 1115  loss=  0.3957  grad_norm= 0.896  lr=1.80e-07  spk_acc=2279/2363
+
                                                      

 92%|█████████▏| 1115/1207 [10:42:26<53:57, 35.19s/it]
 92%|█████████▏| 1116/1207 [10:43:00<52:51, 34.85s/it]
 93%|█████████▎| 1117/1207 [10:43:35<52:19, 34.89s/it]
 93%|█████████▎| 1118/1207 [10:44:12<52:51, 35.63s/it]
 93%|█████████▎| 1119/1207 [10:44:49<52:37, 35.88s/it]
 93%|█████████▎| 1120/1207 [10:45:20<50:13, 34.63s/it]{'loss': '0.3957', 'grad_norm': '0.8961', 'learning_rate': '1.799e-07', 'epoch': '0.9238'}
+step= 1120  loss=  0.3934  grad_norm= 0.790  lr=1.61e-07  spk_acc=2869/3015
+
                                                      

 93%|█████████▎| 1120/1207 [10:45:20<50:13, 34.63s/it]
 93%|█████████▎| 1121/1207 [10:45:58<51:02, 35.62s/it]
 93%|█████████▎| 1122/1207 [10:46:32<49:50, 35.19s/it]
 93%|█████████▎| 1123/1207 [10:47:06<48:26, 34.60s/it]
 93%|█████████▎| 1124/1207 [10:47:44<49:24, 35.72s/it]
 93%|█████████▎| 1125/1207 [10:48:20<48:56, 35.81s/it]{'loss': '0.3934', 'grad_norm': '0.7901', 'learning_rate': '1.611e-07', 'epoch': '0.9279'}
+step= 1125  loss=  0.3752  grad_norm= 0.649  lr=1.43e-07  spk_acc=2784/2906
+
                                                      

 93%|█████████▎| 1125/1207 [10:48:20<48:56, 35.81s/it]
 93%|█████████▎| 1126/1207 [10:48:59<49:46, 36.87s/it]
 93%|█████████▎| 1127/1207 [10:49:34<48:08, 36.11s/it]
 93%|█████████▎| 1128/1207 [10:50:07<46:23, 35.24s/it]
 94%|█████████▎| 1129/1207 [10:50:36<43:27, 33.43s/it]
 94%|█████████▎| 1130/1207 [10:51:10<43:10, 33.64s/it]{'loss': '0.3752', 'grad_norm': '0.649', 'learning_rate': '1.434e-07', 'epoch': '0.9321'}
+step= 1130  loss=  0.3557  grad_norm= 0.781  lr=1.27e-07  spk_acc=2859/2993
+
                                                      

 94%|█████████▎| 1130/1207 [10:51:10<43:10, 33.64s/it]
 94%|█████████▎| 1131/1207 [10:51:46<43:35, 34.41s/it]
 94%|█████████▍| 1132/1207 [10:52:21<42:59, 34.39s/it]
 94%|█████████▍| 1133/1207 [10:52:49<40:00, 32.44s/it]
 94%|█████████▍| 1134/1207 [10:53:20<39:04, 32.12s/it]
 94%|█████████▍| 1135/1207 [10:53:52<38:37, 32.19s/it]{'loss': '0.3557', 'grad_norm': '0.7809', 'learning_rate': '1.267e-07', 'epoch': '0.9362'}
+step= 1135  loss=  0.4033  grad_norm= 0.813  lr=1.11e-07  spk_acc=2916/3040
+
                                                      

 94%|█████████▍| 1135/1207 [10:53:52<38:37, 32.19s/it]
 94%|█████████▍| 1136/1207 [10:54:27<39:04, 33.02s/it]
 94%|█████████▍| 1137/1207 [10:55:16<44:08, 37.83s/it]
 94%|█████████▍| 1138/1207 [10:55:44<40:04, 34.85s/it]
 94%|█████████▍| 1139/1207 [10:56:18<39:06, 34.51s/it]
 94%|█████████▍| 1140/1207 [10:57:04<42:17, 37.87s/it]{'loss': '0.4033', 'grad_norm': '0.8132', 'learning_rate': '1.111e-07', 'epoch': '0.9403'}
+step= 1140  loss=  0.4411  grad_norm= 1.102  lr=9.64e-08  spk_acc=3530/3788
+
                                                      

 94%|█████████▍| 1140/1207 [10:57:04<42:17, 37.87s/it]
 95%|█████████▍| 1141/1207 [10:57:42<41:44, 37.94s/it]
 95%|█████████▍| 1142/1207 [10:58:13<39:02, 36.04s/it]
 95%|█████████▍| 1143/1207 [10:58:39<35:00, 32.82s/it]
 95%|█████████▍| 1144/1207 [10:59:17<36:14, 34.51s/it]
 95%|█████████▍| 1145/1207 [10:59:51<35:33, 34.42s/it]{'loss': '0.4411', 'grad_norm': '1.102', 'learning_rate': '9.643e-08', 'epoch': '0.9445'}
+step= 1145  loss=  0.4177  grad_norm= 0.961  lr=8.28e-08  spk_acc=2731/2953
+
                                                      

 95%|█████████▍| 1145/1207 [10:59:51<35:33, 34.42s/it]
 95%|█████████▍| 1146/1207 [11:00:24<34:28, 33.90s/it]
 95%|█████████▌| 1147/1207 [11:01:00<34:36, 34.60s/it]
 95%|█████████▌| 1148/1207 [11:01:37<34:36, 35.20s/it]
 95%|█████████▌| 1149/1207 [11:02:12<34:02, 35.21s/it]
 95%|█████████▌| 1150/1207 [11:02:44<32:35, 34.31s/it]{'loss': '0.4177', 'grad_norm': '0.9612', 'learning_rate': '8.281e-08', 'epoch': '0.9486'}
+step= 1150  loss=  0.3733  grad_norm= 0.961  lr=7.02e-08  spk_acc=2139/2210
+
                                                      

 95%|█████████▌| 1150/1207 [11:02:44<32:35, 34.31s/it]
 95%|█████████▌| 1151/1207 [11:03:18<31:57, 34.23s/it]
 95%|█████████▌| 1152/1207 [11:03:54<31:47, 34.69s/it]
 96%|█████████▌| 1153/1207 [11:04:28<31:03, 34.51s/it]
 96%|█████████▌| 1154/1207 [11:05:08<31:54, 36.12s/it]
 96%|█████████▌| 1155/1207 [11:05:48<32:20, 37.32s/it]{'loss': '0.3733', 'grad_norm': '0.9607', 'learning_rate': '7.021e-08', 'epoch': '0.9528'}
+step= 1155  loss=  0.4329  grad_norm= 1.114  lr=5.87e-08  spk_acc=3338/3609
+
                                                      

 96%|█████████▌| 1155/1207 [11:05:48<32:20, 37.32s/it]
 96%|█████████▌| 1156/1207 [11:06:19<30:06, 35.41s/it]
 96%|█████████▌| 1157/1207 [11:06:49<28:04, 33.68s/it]
 96%|█████████▌| 1158/1207 [11:07:22<27:23, 33.53s/it]
 96%|█████████▌| 1159/1207 [11:07:56<26:51, 33.56s/it]
 96%|█████████▌| 1160/1207 [11:08:30<26:27, 33.77s/it]{'loss': '0.4329', 'grad_norm': '1.114', 'learning_rate': '5.865e-08', 'epoch': '0.9569'}
+step= 1160  loss=  0.3456  grad_norm= 0.703  lr=4.81e-08  spk_acc=2392/2452
+
                                                      

 96%|█████████▌| 1160/1207 [11:08:30<26:27, 33.77s/it]
 96%|█████████▌| 1161/1207 [11:09:04<25:51, 33.73s/it]
 96%|█████████▋| 1162/1207 [11:09:36<25:01, 33.36s/it]
 96%|█████████▋| 1163/1207 [11:10:08<24:05, 32.85s/it]
 96%|█████████▋| 1164/1207 [11:10:39<23:08, 32.29s/it]
 97%|█████████▋| 1165/1207 [11:11:08<22:02, 31.50s/it]{'loss': '0.3456', 'grad_norm': '0.7033', 'learning_rate': '4.812e-08', 'epoch': '0.9611'}
+step= 1165  loss=  0.4092  grad_norm= 0.626  lr=3.86e-08  spk_acc=2138/2279
+
                                                      

 97%|█████████▋| 1165/1207 [11:11:08<22:02, 31.50s/it]
 97%|█████████▋| 1166/1207 [11:11:41<21:39, 31.71s/it]
 97%|█████████▋| 1167/1207 [11:12:09<20:24, 30.61s/it]
 97%|█████████▋| 1168/1207 [11:12:42<20:23, 31.37s/it]
 97%|█████████▋| 1169/1207 [11:13:16<20:29, 32.36s/it]
 97%|█████████▋| 1170/1207 [11:13:50<20:09, 32.70s/it]{'loss': '0.4092', 'grad_norm': '0.6257', 'learning_rate': '3.863e-08', 'epoch': '0.9652'}
+step= 1170  loss=  0.3628  grad_norm= 0.672  lr=3.02e-08  spk_acc=2291/2351
+
                                                      

 97%|█████████▋| 1170/1207 [11:13:50<20:09, 32.70s/it]
 97%|█████████▋| 1171/1207 [11:14:25<20:01, 33.38s/it]
 97%|█████████▋| 1172/1207 [11:14:59<19:36, 33.62s/it]
 97%|█████████▋| 1173/1207 [11:15:37<19:46, 34.91s/it]
 97%|█████████▋| 1174/1207 [11:16:11<18:59, 34.52s/it]
 97%|█████████▋| 1175/1207 [11:16:44<18:17, 34.29s/it]{'loss': '0.3628', 'grad_norm': '0.6722', 'learning_rate': '3.018e-08', 'epoch': '0.9693'}
+step= 1175  loss=  0.3401  grad_norm= 0.579  lr=2.28e-08  spk_acc=2512/2660
+
                                                      

 97%|█████████▋| 1175/1207 [11:16:44<18:17, 34.29s/it]
 97%|█████████▋| 1176/1207 [11:17:16<17:21, 33.59s/it]
 98%|█████████▊| 1177/1207 [11:17:47<16:24, 32.81s/it]
 98%|█████████▊| 1178/1207 [11:18:20<15:52, 32.86s/it]
 98%|█████████▊| 1179/1207 [11:18:58<15:57, 34.19s/it]
 98%|█████████▊| 1180/1207 [11:19:29<15:01, 33.40s/it]{'loss': '0.3401', 'grad_norm': '0.579', 'learning_rate': '2.277e-08', 'epoch': '0.9735'}
+step= 1180  loss=  0.4264  grad_norm= 1.075  lr=1.64e-08  spk_acc=1845/1917
+
                                                      

 98%|█████████▊| 1180/1207 [11:19:29<15:01, 33.40s/it]
 98%|█████████▊| 1181/1207 [11:20:06<14:56, 34.46s/it]
 98%|█████████▊| 1182/1207 [11:20:38<14:02, 33.69s/it]
 98%|█████████▊| 1183/1207 [11:21:13<13:41, 34.24s/it]
 98%|█████████▊| 1184/1207 [11:21:47<13:03, 34.05s/it]
 98%|█████████▊| 1185/1207 [11:22:21<12:28, 34.04s/it]{'loss': '0.4264', 'grad_norm': '1.075', 'learning_rate': '1.639e-08', 'epoch': '0.9776'}
+step= 1185  loss=  0.3471  grad_norm= 0.541  lr=1.11e-08  spk_acc=2327/2413
+
                                                      

 98%|█████████▊| 1185/1207 [11:22:21<12:28, 34.04s/it]
 98%|█████████▊| 1186/1207 [11:22:53<11:38, 33.27s/it]
 98%|█████████▊| 1187/1207 [11:23:27<11:11, 33.56s/it]
 98%|█████████▊| 1188/1207 [11:24:02<10:47, 34.07s/it]
 99%|█████████▊| 1189/1207 [11:24:44<10:53, 36.33s/it]
 99%|█████████▊| 1190/1207 [11:25:17<10:02, 35.44s/it]{'loss': '0.3471', 'grad_norm': '0.5415', 'learning_rate': '1.106e-08', 'epoch': '0.9818'}
+step= 1190  loss=  0.3521  grad_norm= 0.806  lr=6.78e-09  spk_acc=2294/2392
+
                                                      

 99%|█████████▊| 1190/1207 [11:25:17<10:02, 35.44s/it]
 99%|█████████▊| 1191/1207 [11:25:53<09:28, 35.54s/it]
 99%|█████████▉| 1192/1207 [11:26:26<08:44, 34.95s/it]
 99%|█████████▉| 1193/1207 [11:27:05<08:25, 36.11s/it]
 99%|█████████▉| 1194/1207 [11:27:38<07:36, 35.10s/it]
 99%|█████████▉| 1195/1207 [11:28:12<06:58, 34.87s/it]{'loss': '0.3521', 'grad_norm': '0.8059', 'learning_rate': '6.777e-09', 'epoch': '0.9859'}
+step= 1195  loss=  0.3922  grad_norm= 0.506  lr=3.54e-09  spk_acc=3080/3223
+
                                                      

 99%|█████████▉| 1195/1207 [11:28:12<06:58, 34.87s/it]
 99%|█████████▉| 1196/1207 [11:28:47<06:24, 34.97s/it]
 99%|█████████▉| 1197/1207 [11:29:18<05:36, 33.68s/it]
 99%|█████████▉| 1198/1207 [11:30:00<05:25, 36.22s/it]
 99%|█████████▉| 1199/1207 [11:30:26<04:25, 33.23s/it]
 99%|█████████▉| 1200/1207 [11:31:04<04:02, 34.60s/it]{'loss': '0.3922', 'grad_norm': '0.5058', 'learning_rate': '3.535e-09', 'epoch': '0.9901'}
+step= 1200  loss=  0.4515  grad_norm= 1.068  lr=1.34e-09  spk_acc=3174/3462
+
                                                      

 99%|█████████▉| 1200/1207 [11:31:04<04:02, 34.60s/it]{'loss': '0.4515', 'grad_norm': '1.068', 'learning_rate': '1.339e-09', 'epoch': '0.9942'}
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.16s/it]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.16s/it]
+/wangshuai/moss/MOSS_Speaker-RoPE/moss_speaker_rope/wespeaker_redimnet/tfmel.py:296: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enabled=False):
+
100%|█████████▉| 1201/1207 [11:31:55<03:57, 39.58s/it]
100%|█████████▉| 1202/1207 [11:32:27<03:05, 37.20s/it]
100%|█████████▉| 1203/1207 [11:33:04<02:28, 37.05s/it]
100%|█████████▉| 1204/1207 [11:33:33<01:44, 34.75s/it]
100%|█████████▉| 1205/1207 [11:34:03<01:06, 33.35s/it]step= 1205  loss=  0.3403  grad_norm= 1.689  lr=1.88e-10  spk_acc=1949/1989
+
                                                      

100%|█████████▉| 1205/1207 [11:34:03<01:06, 33.35s/it]
100%|█████████▉| 1206/1207 [11:34:38<00:33, 33.87s/it]
100%|██████████| 1207/1207 [11:35:12<00:00, 33.90s/it]{'loss': '0.3403', 'grad_norm': '1.689', 'learning_rate': '1.883e-10', 'epoch': '0.9983'}
+
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
+
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.72s/it]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.72s/it]
+
                                                      

100%|██████████| 1207/1207 [11:35:19<00:00, 33.90s/it]
100%|██████████| 1207/1207 [11:35:19<00:00, 34.56s/it]
+{'train_runtime': '4.172e+04', 'train_samples_per_second': '0.029', 'train_steps_per_second': '0.029', 'train_loss': '0.3548', 'epoch': '1'}
+
Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.26s/it]
Writing model shards: 100%|██████████| 1/1 [00:02<00:00,  2.26s/it]
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..992babc967a3411672658e64a124bf778bf2e31c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.532391257768261e+17,
+    "train_loss": 0.35478373169405164,
+    "train_runtime": 41719.5034,
+    "train_samples_per_second": 0.029,
+    "train_steps_per_second": 0.029
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a4b73f7eca55ab09cf002d2247146ac9f3b0c3d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/trainer_state.json
@@ -0,0 +1,1730 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 0.9088397026062012,
+      "learning_rate": 3.3057851239669426e-07,
+      "loss": 0.46797571182250974,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 0.4194833040237427,
+      "learning_rate": 7.438016528925621e-07,
+      "loss": 0.32537107467651366,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 0.48602092266082764,
+      "learning_rate": 1.15702479338843e-06,
+      "loss": 0.440777587890625,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 0.615478515625,
+      "learning_rate": 1.5702479338842977e-06,
+      "loss": 0.3342358350753784,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 0.4429086148738861,
+      "learning_rate": 1.9834710743801654e-06,
+      "loss": 0.4068150043487549,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 0.7779952883720398,
+      "learning_rate": 2.3966942148760335e-06,
+      "loss": 0.4107870101928711,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 0.4232267439365387,
+      "learning_rate": 2.809917355371901e-06,
+      "loss": 0.4021468639373779,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 0.5329512357711792,
+      "learning_rate": 3.2231404958677685e-06,
+      "loss": 0.40447416305541994,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 0.40902742743492126,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.4133272647857666,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 0.7031140327453613,
+      "learning_rate": 4.049586776859504e-06,
+      "loss": 0.3559228181838989,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 0.5372095704078674,
+      "learning_rate": 4.462809917355372e-06,
+      "loss": 0.3345926761627197,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.4524230360984802,
+      "learning_rate": 4.87603305785124e-06,
+      "loss": 0.3521779775619507,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.405799001455307,
+      "learning_rate": 5.289256198347108e-06,
+      "loss": 0.32509465217590333,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.6495310068130493,
+      "learning_rate": 5.702479338842976e-06,
+      "loss": 0.25270297527313235,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 0.565504252910614,
+      "learning_rate": 6.115702479338843e-06,
+      "loss": 0.2897182941436768,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 0.9850594401359558,
+      "learning_rate": 6.528925619834712e-06,
+      "loss": 0.35270593166351316,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.590770423412323,
+      "learning_rate": 6.942148760330579e-06,
+      "loss": 0.3310926198959351,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.3464272916316986,
+      "learning_rate": 7.355371900826447e-06,
+      "loss": 0.6234882831573486,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 0.6508628129959106,
+      "learning_rate": 7.768595041322314e-06,
+      "loss": 0.36177287101745603,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 0.4168076515197754,
+      "learning_rate": 8.181818181818183e-06,
+      "loss": 0.37993545532226564,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.4954686462879181,
+      "learning_rate": 8.59504132231405e-06,
+      "loss": 0.32210068702697753,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.3881250321865082,
+      "learning_rate": 9.008264462809918e-06,
+      "loss": 0.34254477024078367,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.4197976291179657,
+      "learning_rate": 9.421487603305785e-06,
+      "loss": 0.30668623447418214,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.4320375621318817,
+      "learning_rate": 9.834710743801654e-06,
+      "loss": 0.29941961765289304,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.4299139380455017,
+      "learning_rate": 9.999811713194481e-06,
+      "loss": 0.3240851402282715,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.5377570986747742,
+      "learning_rate": 9.99866112295902e-06,
+      "loss": 0.34094393253326416,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.34123289585113525,
+      "learning_rate": 9.996464786686645e-06,
+      "loss": 0.28050880432128905,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.4427873492240906,
+      "learning_rate": 9.993223163862385e-06,
+      "loss": 0.34813799858093264,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.359440416097641,
+      "learning_rate": 9.988936932650682e-06,
+      "loss": 0.27970683574676514,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.5566115975379944,
+      "learning_rate": 9.98360698975351e-06,
+      "loss": 0.3594496250152588,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 0.4960085153579712,
+      "learning_rate": 9.977234450222783e-06,
+      "loss": 0.334060525894165,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.5455737709999084,
+      "learning_rate": 9.969820647227086e-06,
+      "loss": 0.3540655136108398,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.42919108271598816,
+      "learning_rate": 9.961367131772759e-06,
+      "loss": 0.29244580268859866,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 1.15133798122406,
+      "learning_rate": 9.951875672379424e-06,
+      "loss": 0.3785684108734131,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.3836948573589325,
+      "learning_rate": 9.941348254710004e-06,
+      "loss": 0.32334723472595217,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.38102057576179504,
+      "learning_rate": 9.929787081155305e-06,
+      "loss": 0.2559232711791992,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.4217875897884369,
+      "learning_rate": 9.91719457037327e-06,
+      "loss": 0.3103822708129883,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.43528667092323303,
+      "learning_rate": 9.903573356782978e-06,
+      "loss": 0.29477834701538086,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.7707864046096802,
+      "learning_rate": 9.888926290013521e-06,
+      "loss": 0.304648232460022,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.34778332710266113,
+      "learning_rate": 9.873256434307828e-06,
+      "loss": 0.27324905395507815,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.34213224053382874,
+      "learning_rate": 9.856567067881628e-06,
+      "loss": 0.22963452339172363,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 0.4351404011249542,
+      "learning_rate": 9.838861682237619e-06,
+      "loss": 0.28716602325439455,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.553987979888916,
+      "learning_rate": 9.820143981435031e-06,
+      "loss": 0.26800360679626467,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.49172210693359375,
+      "learning_rate": 9.800417881314717e-06,
+      "loss": 0.31684017181396484,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.36292967200279236,
+      "learning_rate": 9.779687508679936e-06,
+      "loss": 0.3675337791442871,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.5972428917884827,
+      "learning_rate": 9.757957200433011e-06,
+      "loss": 0.30315725803375243,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.6428586840629578,
+      "learning_rate": 9.73523150266802e-06,
+      "loss": 0.321216082572937,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.4479713439941406,
+      "learning_rate": 9.711515169719732e-06,
+      "loss": 0.3546335458755493,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6334832906723022,
+      "learning_rate": 9.686813163168973e-06,
+      "loss": 0.26909546852111815,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.49879494309425354,
+      "learning_rate": 9.661130650804643e-06,
+      "loss": 0.269888710975647,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.4971435070037842,
+      "learning_rate": 9.634473005542579e-06,
+      "loss": 0.3372554063796997,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5778576135635376,
+      "learning_rate": 9.606845804301523e-06,
+      "loss": 0.3382244348526001,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.5813653469085693,
+      "learning_rate": 9.57825482683639e-06,
+      "loss": 0.2622408866882324,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.5511571764945984,
+      "learning_rate": 9.548706054529124e-06,
+      "loss": 0.3133584499359131,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 0.5167928338050842,
+      "learning_rate": 9.518205669137353e-06,
+      "loss": 0.24319915771484374,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.49302011728286743,
+      "learning_rate": 9.486760051501139e-06,
+      "loss": 0.3253422021865845,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.46096402406692505,
+      "learning_rate": 9.454375780208072e-06,
+      "loss": 0.3033352613449097,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.29232358932495117,
+      "learning_rate": 9.421059630216992e-06,
+      "loss": 0.28919391632080077,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.37558865547180176,
+      "learning_rate": 9.386818571440631e-06,
+      "loss": 0.2719090700149536,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.34535545110702515,
+      "learning_rate": 9.351659767287477e-06,
+      "loss": 0.32380578517913816,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.38860467076301575,
+      "learning_rate": 9.315590573163152e-06,
+      "loss": 0.32611970901489257,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.4789179861545563,
+      "learning_rate": 9.278618534931612e-06,
+      "loss": 0.31763691902160646,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.5553998947143555,
+      "learning_rate": 9.240751387336522e-06,
+      "loss": 0.3328229904174805,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.54727703332901,
+      "learning_rate": 9.201997052383107e-06,
+      "loss": 0.34824743270874026,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.6511346101760864,
+      "learning_rate": 9.162363637680822e-06,
+      "loss": 0.2723949432373047,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.5212080478668213,
+      "learning_rate": 9.121859434747205e-06,
+      "loss": 0.34482150077819823,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 0.8779405355453491,
+      "learning_rate": 9.080492917273238e-06,
+      "loss": 0.3662183046340942,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.8695563077926636,
+      "learning_rate": 9.038272739350617e-06,
+      "loss": 0.2668049097061157,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.4466971755027771,
+      "learning_rate": 8.995207733661266e-06,
+      "loss": 0.26048974990844725,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.4294697344303131,
+      "learning_rate": 8.951306909629492e-06,
+      "loss": 0.26141631603240967,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.38591551780700684,
+      "learning_rate": 8.906579451537172e-06,
+      "loss": 0.25841057300567627,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 0.6442238688468933,
+      "learning_rate": 8.861034716602343e-06,
+      "loss": 0.3345190525054932,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.4142279028892517,
+      "learning_rate": 8.814682233021636e-06,
+      "loss": 0.2878074645996094,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.46708178520202637,
+      "learning_rate": 8.76753169797692e-06,
+      "loss": 0.2928623676300049,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 1.1409103870391846,
+      "learning_rate": 8.719592975606601e-06,
+      "loss": 0.2992920637130737,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.9347559213638306,
+      "learning_rate": 8.670876094941991e-06,
+      "loss": 0.33010694980621336,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.4922148585319519,
+      "learning_rate": 8.62139124780919e-06,
+      "loss": 0.3573878288269043,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.47879040241241455,
+      "learning_rate": 8.571148786696894e-06,
+      "loss": 0.29300789833068847,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.44266507029533386,
+      "learning_rate": 8.520159222590605e-06,
+      "loss": 0.27748234272003175,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.5809404850006104,
+      "learning_rate": 8.468433222773677e-06,
+      "loss": 0.3408142328262329,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.49024492502212524,
+      "learning_rate": 8.415981608595666e-06,
+      "loss": 0.291151237487793,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.40204304456710815,
+      "learning_rate": 8.362815353208441e-06,
+      "loss": 0.2749445915222168,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.5574970841407776,
+      "learning_rate": 8.308945579270551e-06,
+      "loss": 0.33534510135650636,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.7230126261711121,
+      "learning_rate": 8.2543835566203e-06,
+      "loss": 0.3234541416168213,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6410138010978699,
+      "learning_rate": 8.19914069991805e-06,
+      "loss": 0.323453688621521,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.689175009727478,
+      "learning_rate": 8.143228566258192e-06,
+      "loss": 0.29622983932495117,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.4988920986652374,
+      "learning_rate": 8.08665885275137e-06,
+      "loss": 0.3264308929443359,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.5350947976112366,
+      "learning_rate": 8.029443394077356e-06,
+      "loss": 0.32159135341644285,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.5879245400428772,
+      "learning_rate": 7.971594160009195e-06,
+      "loss": 0.35001649856567385,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.6696264147758484,
+      "learning_rate": 7.913123252909056e-06,
+      "loss": 0.30917251110076904,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.6794363260269165,
+      "learning_rate": 7.854042905196355e-06,
+      "loss": 0.34172370433807375,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.3615648150444031,
+      "learning_rate": 7.794365476788668e-06,
+      "loss": 0.26951897144317627,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.4700421988964081,
+      "learning_rate": 7.734103452515976e-06,
+      "loss": 0.33142666816711425,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.452709436416626,
+      "learning_rate": 7.67326943950877e-06,
+      "loss": 0.3427105903625488,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.48398926854133606,
+      "learning_rate": 7.611876164560578e-06,
+      "loss": 0.24839515686035157,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.2996172308921814,
+      "learning_rate": 7.549936471465443e-06,
+      "loss": 0.2821991205215454,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4082246720790863,
+      "learning_rate": 7.487463318330945e-06,
+      "loss": 0.2629000425338745,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.6337223052978516,
+      "learning_rate": 7.424469774867291e-06,
+      "loss": 0.25711946487426757,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.7650140523910522,
+      "learning_rate": 7.360969019653068e-06,
+      "loss": 0.4765789031982422,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.6438851356506348,
+      "learning_rate": 7.296974337378209e-06,
+      "loss": 0.27527151107788084,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 1.112627387046814,
+      "learning_rate": 7.2324991160647705e-06,
+      "loss": 0.2819572925567627,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 0.9886193871498108,
+      "learning_rate": 7.1675568442660915e-06,
+      "loss": 0.3160847663879395,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.5070939660072327,
+      "learning_rate": 7.102161108244907e-06,
+      "loss": 0.2885098934173584,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.5902522206306458,
+      "learning_rate": 7.036325589131048e-06,
+      "loss": 0.3293052911758423,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.5462592244148254,
+      "learning_rate": 6.9700640600592585e-06,
+      "loss": 0.28150961399078367,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.48972848057746887,
+      "learning_rate": 6.903390383287795e-06,
+      "loss": 0.2742903232574463,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.6057692766189575,
+      "learning_rate": 6.8363185072983695e-06,
+      "loss": 0.3347393751144409,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.45753175020217896,
+      "learning_rate": 6.7688624638780475e-06,
+      "loss": 0.32706599235534667,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.7748094797134399,
+      "learning_rate": 6.701036365183737e-06,
+      "loss": 0.38280029296875,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.589371383190155,
+      "learning_rate": 6.632854400789835e-06,
+      "loss": 0.3170490264892578,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.5430629849433899,
+      "learning_rate": 6.5643308347197085e-06,
+      "loss": 0.3212059259414673,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.4012330174446106,
+      "learning_rate": 6.495480002461577e-06,
+      "loss": 0.31148293018341067,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.5642695426940918,
+      "learning_rate": 6.4263163079694645e-06,
+      "loss": 0.3561388969421387,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.5468930006027222,
+      "learning_rate": 6.356854220649801e-06,
+      "loss": 0.32206296920776367,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.6053391098976135,
+      "learning_rate": 6.28710827233436e-06,
+      "loss": 0.326386547088623,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.7525632977485657,
+      "learning_rate": 6.21709305424012e-06,
+      "loss": 0.3004195213317871,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.5053286552429199,
+      "learning_rate": 6.146823213916703e-06,
+      "loss": 0.3025178909301758,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.37086620926856995,
+      "learning_rate": 6.076313452182033e-06,
+      "loss": 0.3551692247390747,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.570372462272644,
+      "learning_rate": 6.005578520046848e-06,
+      "loss": 0.6431361198425293,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.3969172239303589,
+      "learning_rate": 5.934633215628711e-06,
+      "loss": 0.31665098667144775,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.5713177919387817,
+      "learning_rate": 5.863492381056164e-06,
+      "loss": 0.29021525382995605,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5128471255302429,
+      "learning_rate": 5.792170899363683e-06,
+      "loss": 0.33125274181365966,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.5780979990959167,
+      "learning_rate": 5.720683691378061e-06,
+      "loss": 0.29233403205871583,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 0.6340335011482239,
+      "learning_rate": 5.6490457125969035e-06,
+      "loss": 0.27084898948669434,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.5844733715057373,
+      "learning_rate": 5.57727195005985e-06,
+      "loss": 0.3200614213943481,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.644882082939148,
+      "learning_rate": 5.505377419213214e-06,
+      "loss": 0.32254467010498045,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.47608786821365356,
+      "learning_rate": 5.433377160768669e-06,
+      "loss": 0.3473681449890137,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.6619153022766113,
+      "learning_rate": 5.361286237556658e-06,
+      "loss": 0.3421504020690918,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.6047444343566895,
+      "learning_rate": 5.289119731375161e-06,
+      "loss": 0.3683443307876587,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 1.0079972743988037,
+      "learning_rate": 5.216892739834519e-06,
+      "loss": 0.3449583053588867,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.5912438035011292,
+      "learning_rate": 5.144620373198916e-06,
+      "loss": 0.38611855506896975,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.6063432693481445,
+      "learning_rate": 5.072317751225254e-06,
+      "loss": 0.37934141159057616,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.4390994906425476,
+      "learning_rate": 5e-06,
+      "loss": 0.27857556343078616,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.6064956784248352,
+      "learning_rate": 4.927682248774747e-06,
+      "loss": 0.3554774284362793,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.6314171552658081,
+      "learning_rate": 4.855379626801085e-06,
+      "loss": 0.30284528732299804,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.671026349067688,
+      "learning_rate": 4.783107260165483e-06,
+      "loss": 0.36607491970062256,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.4459201395511627,
+      "learning_rate": 4.710880268624841e-06,
+      "loss": 0.3821887016296387,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.6088168621063232,
+      "learning_rate": 4.6387137624433446e-06,
+      "loss": 0.3565082550048828,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.5071287751197815,
+      "learning_rate": 4.5666228392313315e-06,
+      "loss": 0.33942298889160155,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.7531337141990662,
+      "learning_rate": 4.494622580786788e-06,
+      "loss": 0.3676978588104248,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.5007270574569702,
+      "learning_rate": 4.422728049940151e-06,
+      "loss": 0.28489739894866944,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.42371106147766113,
+      "learning_rate": 4.350954287403099e-06,
+      "loss": 0.2984581232070923,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.563149631023407,
+      "learning_rate": 4.279316308621941e-06,
+      "loss": 0.40024795532226565,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 0.8423017263412476,
+      "learning_rate": 4.207829100636318e-06,
+      "loss": 0.3361294507980347,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.5179011821746826,
+      "learning_rate": 4.1365076189438365e-06,
+      "loss": 0.3160973072052002,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.7851564288139343,
+      "learning_rate": 4.06536678437129e-06,
+      "loss": 0.36293606758117675,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5663831233978271,
+      "learning_rate": 3.994421479953153e-06,
+      "loss": 0.3758086681365967,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.6246234178543091,
+      "learning_rate": 3.9236865478179685e-06,
+      "loss": 0.29956531524658203,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.5657338500022888,
+      "learning_rate": 3.853176786083299e-06,
+      "loss": 0.37062208652496337,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 0.780840277671814,
+      "learning_rate": 3.782906945759883e-06,
+      "loss": 0.38033413887023926,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.46465229988098145,
+      "learning_rate": 3.7128917276656406e-06,
+      "loss": 0.2972914218902588,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.6549701690673828,
+      "learning_rate": 3.6431457793502e-06,
+      "loss": 0.38214161396026614,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.7952358722686768,
+      "learning_rate": 3.5736836920305367e-06,
+      "loss": 0.43451919555664065,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.3544764220714569,
+      "learning_rate": 3.5045199975384225e-06,
+      "loss": 0.28854324817657473,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5664554834365845,
+      "learning_rate": 3.4356691652802936e-06,
+      "loss": 0.36930935382843016,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.46652841567993164,
+      "learning_rate": 3.3671455992101665e-06,
+      "loss": 0.3794795036315918,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.5960412621498108,
+      "learning_rate": 3.2989636348162633e-06,
+      "loss": 0.6944560050964356,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.5001474022865295,
+      "learning_rate": 3.2311375361219538e-06,
+      "loss": 0.35518996715545653,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.36455702781677246,
+      "learning_rate": 3.163681492701632e-06,
+      "loss": 0.3554563522338867,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.578860342502594,
+      "learning_rate": 3.096609616712207e-06,
+      "loss": 0.4234604835510254,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 0.6481260657310486,
+      "learning_rate": 3.0299359399407436e-06,
+      "loss": 0.34542040824890136,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 1.1381938457489014,
+      "learning_rate": 2.963674410868953e-06,
+      "loss": 0.5298830032348633,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.6089155077934265,
+      "learning_rate": 2.8978388917550936e-06,
+      "loss": 0.33658499717712403,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.7562251687049866,
+      "learning_rate": 2.8324431557339106e-06,
+      "loss": 0.3880928516387939,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.7808563113212585,
+      "learning_rate": 2.767500883935231e-06,
+      "loss": 0.3246309280395508,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.8138899803161621,
+      "learning_rate": 2.7030256626217932e-06,
+      "loss": 0.4132601261138916,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 0.9273833632469177,
+      "learning_rate": 2.6390309803469326e-06,
+      "loss": 0.4071033477783203,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.5960099101066589,
+      "learning_rate": 2.5755302251327097e-06,
+      "loss": 0.4031569480895996,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.6419629454612732,
+      "learning_rate": 2.5125366816690557e-06,
+      "loss": 0.3036700487136841,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.6673698425292969,
+      "learning_rate": 2.450063528534558e-06,
+      "loss": 0.311791729927063,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.6572533249855042,
+      "learning_rate": 2.3881238354394236e-06,
+      "loss": 0.38104662895202634,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.46291619539260864,
+      "learning_rate": 2.32673056049123e-06,
+      "loss": 0.4063735008239746,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 1.104805827140808,
+      "learning_rate": 2.265896547484026e-06,
+      "loss": 0.36424479484558103,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.6098979115486145,
+      "learning_rate": 2.2056345232113352e-06,
+      "loss": 0.350197958946228,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5803428292274475,
+      "learning_rate": 2.1459570948036486e-06,
+      "loss": 0.364992094039917,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.7771470546722412,
+      "learning_rate": 2.086876747090945e-06,
+      "loss": 0.38045494556427,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.43313416838645935,
+      "learning_rate": 2.0284058399908045e-06,
+      "loss": 0.3114879369735718,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.5160917639732361,
+      "learning_rate": 1.970556605922645e-06,
+      "loss": 0.41294088363647463,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.089125633239746,
+      "learning_rate": 1.9133411472486324e-06,
+      "loss": 0.45021257400512693,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.6227880716323853,
+      "learning_rate": 1.8567714337418092e-06,
+      "loss": 0.3855113506317139,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.8897408246994019,
+      "learning_rate": 1.8008593000819519e-06,
+      "loss": 0.42201600074768064,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.4485880434513092,
+      "learning_rate": 1.7456164433796991e-06,
+      "loss": 0.39100966453552244,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.5533657670021057,
+      "learning_rate": 1.6910544207294521e-06,
+      "loss": 0.33871314525604246,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.6669449806213379,
+      "learning_rate": 1.6371846467915603e-06,
+      "loss": 0.35631103515625,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.6646720170974731,
+      "learning_rate": 1.5840183914043338e-06,
+      "loss": 0.3687641143798828,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.9954160451889038,
+      "learning_rate": 1.5315667772263227e-06,
+      "loss": 0.4456006050109863,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.6651373505592346,
+      "learning_rate": 1.4798407774093954e-06,
+      "loss": 0.3330822467803955,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.8815678954124451,
+      "learning_rate": 1.4288512133031073e-06,
+      "loss": 0.4016695022583008,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.6688061952590942,
+      "learning_rate": 1.3786087521908103e-06,
+      "loss": 0.3567208290100098,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 2.1051671504974365,
+      "learning_rate": 1.3291239050580085e-06,
+      "loss": 0.6473428726196289,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.9308188557624817,
+      "learning_rate": 1.2804070243933997e-06,
+      "loss": 0.3749030590057373,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.7065919637680054,
+      "learning_rate": 1.2324683020230798e-06,
+      "loss": 0.733770227432251,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.7088451981544495,
+      "learning_rate": 1.1853177669783645e-06,
+      "loss": 0.37824020385742185,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.7551929950714111,
+      "learning_rate": 1.1389652833976578e-06,
+      "loss": 0.4163835048675537,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.5236889719963074,
+      "learning_rate": 1.0934205484628285e-06,
+      "loss": 0.39182589054107664,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.5291634202003479,
+      "learning_rate": 1.0486930903705095e-06,
+      "loss": 0.3792252540588379,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6111361384391785,
+      "learning_rate": 1.004792266338735e-06,
+      "loss": 0.4349212169647217,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.6383189558982849,
+      "learning_rate": 9.617272606493844e-07,
+      "loss": 0.3830833435058594,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.6607468128204346,
+      "learning_rate": 9.195070827267633e-07,
+      "loss": 0.4139139175415039,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.9052505493164062,
+      "learning_rate": 8.781405652527975e-07,
+      "loss": 0.38831195831298826,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 1.0455974340438843,
+      "learning_rate": 8.376363623191791e-07,
+      "loss": 0.4037032127380371,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.6647090911865234,
+      "learning_rate": 7.980029476168943e-07,
+      "loss": 0.4418759346008301,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.7218889594078064,
+      "learning_rate": 7.59248612663479e-07,
+      "loss": 0.3176127910614014,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 1.124987006187439,
+      "learning_rate": 7.213814650683887e-07,
+      "loss": 0.484192419052124,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.5644972920417786,
+      "learning_rate": 6.844094268368484e-07,
+      "loss": 0.38383920192718507,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.8091310858726501,
+      "learning_rate": 6.483402327125232e-07,
+      "loss": 0.45018877983093264,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.4238279461860657,
+      "learning_rate": 6.131814285593707e-07,
+      "loss": 0.37433390617370604,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.5011367797851562,
+      "learning_rate": 5.789403697830104e-07,
+      "loss": 0.43130178451538087,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 0.9677279591560364,
+      "learning_rate": 5.456242197919292e-07,
+      "loss": 0.43402600288391113,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.6384739875793457,
+      "learning_rate": 5.132399484988615e-07,
+      "loss": 0.41501555442810056,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.43836286664009094,
+      "learning_rate": 4.817943308626488e-07,
+      "loss": 0.3939836025238037,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.7905747890472412,
+      "learning_rate": 4.5129394547087756e-07,
+      "loss": 0.4381608009338379,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.8479343056678772,
+      "learning_rate": 4.2174517316361075e-07,
+      "loss": 0.42328357696533203,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4482978284358978,
+      "learning_rate": 3.93154195698478e-07,
+      "loss": 0.43547234535217283,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.4706050455570221,
+      "learning_rate": 3.6552699445742026e-07,
+      "loss": 0.4107391834259033,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.6657713055610657,
+      "learning_rate": 3.3886934919535765e-07,
+      "loss": 0.4677100658416748,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.6160376071929932,
+      "learning_rate": 3.131868368310276e-07,
+      "loss": 0.3558232307434082,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.8466643691062927,
+      "learning_rate": 2.8848483028026896e-07,
+      "loss": 0.46170730590820314,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 0.8067461848258972,
+      "learning_rate": 2.647684973319814e-07,
+      "loss": 0.3582716703414917,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.5093105435371399,
+      "learning_rate": 2.4204279956698994e-07,
+      "loss": 0.3920139312744141,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.7828852534294128,
+      "learning_rate": 2.2031249132006548e-07,
+      "loss": 0.4337261199951172,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.9911346435546875,
+      "learning_rate": 1.9958211868528478e-07,
+      "loss": 0.38247008323669435,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.8960981369018555,
+      "learning_rate": 1.7985601856496947e-07,
+      "loss": 0.3957221508026123,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.7901101112365723,
+      "learning_rate": 1.6113831776238099e-07,
+      "loss": 0.39339218139648435,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.6490103602409363,
+      "learning_rate": 1.4343293211837196e-07,
+      "loss": 0.3751667022705078,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.7808877229690552,
+      "learning_rate": 1.2674356569217282e-07,
+      "loss": 0.355701208114624,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.8131973147392273,
+      "learning_rate": 1.110737099864806e-07,
+      "loss": 0.4032705783843994,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 1.1023856401443481,
+      "learning_rate": 9.642664321702188e-08,
+      "loss": 0.4411326885223389,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.9611843228340149,
+      "learning_rate": 8.280542962673166e-08,
+      "loss": 0.4177103996276855,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.9607189893722534,
+      "learning_rate": 7.02129188446965e-08,
+      "loss": 0.3733290910720825,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 1.1137757301330566,
+      "learning_rate": 5.8651745289997065e-08,
+      "loss": 0.4329118251800537,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.7032621502876282,
+      "learning_rate": 4.8124327620576726e-08,
+      "loss": 0.345641565322876,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.6256678700447083,
+      "learning_rate": 3.863286822724244e-08,
+      "loss": 0.409209156036377,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.6722251772880554,
+      "learning_rate": 3.017935277291495e-08,
+      "loss": 0.3628429889678955,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.5789678692817688,
+      "learning_rate": 2.2765549777217787e-08,
+      "loss": 0.34007580280303956,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 1.0749714374542236,
+      "learning_rate": 1.639301024649209e-08,
+      "loss": 0.4264070510864258,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.5414615273475647,
+      "learning_rate": 1.1063067349318946e-08,
+      "loss": 0.34705543518066406,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.8058632612228394,
+      "learning_rate": 6.7768361376152616e-09,
+      "loss": 0.35213332176208495,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.5058417916297913,
+      "learning_rate": 3.535213313355934e-09,
+      "loss": 0.39220623970031737,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 1.0682101249694824,
+      "learning_rate": 1.3388770409805685e-09,
+      "loss": 0.45152854919433594,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.6894797086715698,
+      "learning_rate": 1.8828680551918887e-10,
+      "loss": 0.3403477668762207,
+      "step": 1205
+    },
+    {
+      "epoch": 1.0,
+      "step": 1207,
+      "total_flos": 2.532391257768261e+17,
+      "train_loss": 0.35478373169405164,
+      "train_runtime": 41719.5034,
+      "train_samples_per_second": 0.029,
+      "train_steps_per_second": 0.029
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 400,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.532391257768261e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/training_args.bin b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..649ae980fcab789df31d5926f8d4ba0e31d75659
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71371688dfd537b126d4621e611796dedc057539dcf7397af2bc1955bebc8155
+size 5265
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/all_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/all_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ed41ef7c778cbd27edc07726367ac63287f0d51
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/all_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.5323923576872166e+17,
+    "train_loss": 0.4422070802295119,
+    "train_runtime": 20343.89,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.059
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..46df614de96a6a217670c8735cc745b47778a5e3
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1200/trainer_state.json
@@ -0,0 +1,1714 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9942004971002486,
+  "eval_steps": 500,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 31.01896095275879,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 1.1710086822509767,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 38.59324264526367,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.7715087890625,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.205821514129639,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.7687691211700439,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 2.734048843383789,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.49406042098999026,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.970097780227661,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.5342318058013916,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 3.3853213787078857,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.5030611991882324,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 1.3471293449401855,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.5010397434234619,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 1.2355350255966187,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.5080785274505615,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 1.4255725145339966,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4955380916595459,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 1.175504207611084,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4213558673858643,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 1.0350234508514404,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.4653134346008301,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.8828385472297668,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.46036972999572756,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.5872329473495483,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.4022225379943848,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.9258399605751038,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37816824913024905,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 1.0011563301086426,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.40845155715942383,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.2110610008239746,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.46489362716674804,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.9312787652015686,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.4639500617980957,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.6982877850532532,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7971051692962646,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.2401320934295654,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.48281636238098147,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 1.1120147705078125,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.5136648178100586,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.9892898797988892,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.4707803249359131,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6651239991188049,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48854589462280273,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8432604670524597,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.46013813018798827,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.9123105406761169,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4647829055786133,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8377557992935181,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.4752926826477051,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.7835660576820374,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4945652961730957,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.689082682132721,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4578232765197754,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.8787635564804077,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.5014208316802978,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5683997273445129,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.4341254234313965,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.8870652914047241,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5459663391113281,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 1.156484842300415,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.4926918983459473,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.7714535593986511,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.53131103515625,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7258591651916504,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.45943522453308105,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.2155957221984863,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.6043071746826172,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6804623007774353,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.4359785556793213,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.45761024951934814,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3847378730773926,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.8509073853492737,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4396988391876221,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.8098800778388977,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.4133442401885986,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.7294316291809082,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.47240419387817384,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.7199881076812744,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40730462074279783,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.5415948629379272,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.36444683074951173,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 1.285866618156433,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.42740211486816404,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6586265563964844,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4160896301269531,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.6770569086074829,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5104285717010498,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.640552818775177,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5205618858337402,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.9050871729850769,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.4352677345275879,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.756203293800354,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.447373104095459,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6095989942550659,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.5091766357421875,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6111555099487305,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.41835684776306153,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.7675306797027588,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.47201828956604003,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6159079670906067,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5127533912658692,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5745006203651428,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48856701850891116,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6177580952644348,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39819822311401365,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7953127026557922,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.4401564598083496,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 1.065096378326416,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.40187416076660154,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6611851453781128,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4557036399841309,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7707319855690002,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.4707478046417236,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3965870440006256,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.41887879371643066,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.6925886869430542,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39414989948272705,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4225500822067261,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46974954605102537,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5246486663818359,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4359858512878418,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.7797112464904785,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4812413215637207,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.7133896946907043,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.4543005466461182,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.8236000537872314,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48366827964782716,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.6479238867759705,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4138167381286621,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.7929301857948303,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.48261222839355467,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.362284779548645,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.534283971786499,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.658808708190918,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.41163101196289065,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.516070544719696,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.4010306358337402,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6887349486351013,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40149264335632323,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.5056571960449219,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3919940710067749,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.0694760084152222,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4507146835327148,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6502102017402649,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.4376358985900879,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.5950688719749451,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4519078254699707,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5176505446434021,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.4313666820526123,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3828600347042084,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.44180989265441895,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6753257513046265,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5487648963928222,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6115583181381226,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4736178398132324,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.40781980752944946,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.3815795421600342,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6663995385169983,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.46039581298828125,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5568227171897888,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42682709693908694,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5221118927001953,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.396131706237793,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8339316844940186,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4795412540435791,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6504913568496704,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.4785301208496094,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6242275238037109,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4487310409545898,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6439151167869568,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.4335296630859375,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6024753451347351,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4808655738830566,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.686862587928772,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4313629627227783,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7613002061843872,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4753777503967285,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7629942297935486,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.4324016571044922,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8611165285110474,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46132574081420896,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.43353787064552307,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.373333215713501,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.49695688486099243,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.48189077377319334,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.515783965587616,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.4596214771270752,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.6876562237739563,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.35883824825286864,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.42947614192962646,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42058372497558594,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4379066824913025,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3526319026947021,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.5771088600158691,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.37990965843200686,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.3974665403366089,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6483776569366455,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9839111566543579,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.40949182510375975,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4381893575191498,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3897416353225708,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.350216269493103,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.43353681564331054,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.48635414242744446,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.40407662391662597,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.7840670943260193,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.4466562271118164,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6072182655334473,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.396435022354126,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.5676800012588501,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.37458391189575196,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.7189476490020752,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4634994983673096,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.42080748081207275,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43068742752075195,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.7519166469573975,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5017862319946289,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6198945045471191,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4238149166107178,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3814507722854614,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4124739170074463,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.39002591371536255,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4085509777069092,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.6172817349433899,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4789612293243408,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.6391016244888306,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.4075304985046387,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.594118595123291,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4542539596557617,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9341249465942383,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42259893417358396,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.7288709282875061,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.42269001007080076,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.5448115468025208,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45872178077697756,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5901434421539307,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7925439834594726,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43836620450019836,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4136178970336914,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.49646082520484924,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.3827540159225464,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5691694021224976,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4286805152893066,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.7288464307785034,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3853081226348877,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 1.2016069889068604,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.402403450012207,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.5269529819488525,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.41291465759277346,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.7392662167549133,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.4353643417358398,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.8992120027542114,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.45477585792541503,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.626833975315094,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43680391311645506,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.49689677357673645,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4409040451049805,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6282675266265869,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42667112350463865,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.7126922607421875,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45916152000427246,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5580125451087952,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4701026439666748,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.43736886978149414,
+      "learning_rate": 5e-05,
+      "loss": 0.3640265941619873,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.8759464621543884,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44643449783325195,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4398824870586395,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.3691298484802246,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.562538206577301,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.47481765747070315,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.4328842759132385,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.4613480567932129,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5684780478477478,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.42593884468078613,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.519466757774353,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4044198513031006,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.8425558805465698,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42935881614685056,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.7469388842582703,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35210604667663575,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5494261980056763,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36253814697265624,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.4391236901283264,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4758582592010498,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.7353135347366333,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3888124942779541,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.6122128367424011,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.3988348960876465,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8968481421470642,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.4200006008148193,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5183577537536621,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43859076499938965,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5962955951690674,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.3618325710296631,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.6844689249992371,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.419573450088501,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 2.3041207790374756,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43375449180603026,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.3782942593097687,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.34841461181640626,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5362056493759155,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.4388935089111328,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.8679549694061279,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48665695190429686,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.38520702719688416,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31835892200469973,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5709502696990967,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.4383851051330566,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.513369083404541,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.41875276565551756,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.5494701862335205,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7724744319915772,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.4949307143688202,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.39315123558044435,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.32247358560562134,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.40227518081665037,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5881837606430054,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4585836887359619,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.0115292072296143,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.36199212074279785,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9708449244499207,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5711725234985352,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.5810522437095642,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3762980937957764,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.9754661321640015,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43689351081848143,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.7655085325241089,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554137945175171,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6773413419723511,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.45856328010559083,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.4474434852600098,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.4327108860015869,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4549546241760254,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4396686553955078,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5432566404342651,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.3287363529205322,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.6105192303657532,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3461976766586304,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5887757539749146,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.4058669090270996,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3826632797718048,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.4288635730743408,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7107186913490295,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38454523086547854,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.6407152414321899,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.3689358472824097,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5461570620536804,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.38851041793823243,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6236541271209717,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40999202728271483,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4176172614097595,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.3426840305328369,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.44086140394210815,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4143134593963623,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.6073216199874878,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.46999621391296387,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5244797468185425,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.39008255004882814,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6138545274734497,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.43625812530517577,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5565832257270813,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39960579872131347,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.49259600043296814,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3722024440765381,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.546994686126709,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3682907819747925,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.478095144033432,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.37905890941619874,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5770573616027832,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.45574336051940917,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5589016675949097,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.3476081609725952,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6209889054298401,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.3975348949432373,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.428181529045105,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.37008640766143797,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.5961155295372009,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.6844874858856201,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.638836681842804,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3731102228164673,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.7055111527442932,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7501702308654785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.610455334186554,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024374008178711,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.612060546875,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4162275791168213,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.468790203332901,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.3881744384765625,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.7208874225616455,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.39170546531677247,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6513557434082031,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.4391339302062988,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.45648840069770813,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39624121189117434,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.697978675365448,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4661447525024414,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.753243088722229,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.394376540184021,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.7163575887680054,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4060837268829346,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.7665302753448486,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.432415246963501,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6077792048454285,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.3501024007797241,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.882154643535614,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47015953063964844,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.423498272895813,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.387217116355896,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.5810433030128479,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44982328414916994,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.39362671971321106,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3725650548934937,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.41626113653182983,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44669065475463865,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 1.006791353225708,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4457366466522217,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.502176821231842,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.42588119506835936,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4030306935310364,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.4012446403503418,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.573884904384613,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.44045190811157225,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.4829924404621124,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.4151927947998047,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4790762960910797,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4421385288238525,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.36565345525741577,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.41750688552856446,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.724289059638977,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4673739433288574,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.9092391729354858,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.3644559860229492,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.679870069026947,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.4468817710876465,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.737008810043335,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.35079207420349123,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.597590446472168,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.40719895362854003,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5171096920967102,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42395763397216796,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.9238699078559875,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.38481290340423585,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5502796173095703,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.40758719444274905,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5856905579566956,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41455903053283694,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.4448857605457306,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3878758907318115,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.42990583181381226,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3641160011291504,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6104210019111633,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.4024786472320557,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8527662754058838,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.452549409866333,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6835286021232605,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42240447998046876,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6754205226898193,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.38313636779785154,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 1.0334895849227905,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42669315338134767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5506278872489929,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.36047968864440916,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5626630187034607,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.41873970031738283,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5406614542007446,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.3593956708908081,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.605750560760498,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.3540114641189575,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.5964953899383545,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.4429689884185791,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6756887435913086,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36358070373535156,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.49958691000938416,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.3598761081695557,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.42558741569519043,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.39934353828430175,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 1.0632734298706055,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.440138053894043,
+      "step": 1200
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.518258444938741e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/chat_template.jinja b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..9211db1411b2480fd0698b12fb1b34d08449e9f6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/chat_template.jinja
@@ -0,0 +1,103 @@
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- else -%}
+        {%- for item in content -%}
+            {%- if item.type == 'audio' or 'audio' in item or 'audio_url' in item -%}
+                {{- '<|audio_start|><|audio_pad|><|audio_end|>\n' -}}
+            {%- elif item.type == 'text' -%}
+                {{- item.text -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- render_content(messages[0].content) + '\n\n' }}
+    {%- else %}
+        {{- 'You are a helpful assistant.\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + render_content(messages[0].content) + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set content = render_content(message.content) %}
+    {%- if ns.multi_step_tool and message.role == "user" and content is string and not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content) %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..569326570d2ad937482b4daa5a706a056856d83f
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/checkpoint-1207/trainer_state.json
@@ -0,0 +1,1721 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 31.01896095275879,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 1.1710086822509767,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 38.59324264526367,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.7715087890625,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.205821514129639,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.7687691211700439,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 2.734048843383789,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.49406042098999026,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.970097780227661,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.5342318058013916,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 3.3853213787078857,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.5030611991882324,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 1.3471293449401855,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.5010397434234619,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 1.2355350255966187,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.5080785274505615,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 1.4255725145339966,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4955380916595459,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 1.175504207611084,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4213558673858643,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 1.0350234508514404,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.4653134346008301,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.8828385472297668,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.46036972999572756,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.5872329473495483,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.4022225379943848,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.9258399605751038,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37816824913024905,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 1.0011563301086426,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.40845155715942383,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.2110610008239746,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.46489362716674804,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.9312787652015686,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.4639500617980957,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.6982877850532532,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7971051692962646,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.2401320934295654,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.48281636238098147,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 1.1120147705078125,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.5136648178100586,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.9892898797988892,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.4707803249359131,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6651239991188049,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48854589462280273,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8432604670524597,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.46013813018798827,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.9123105406761169,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4647829055786133,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8377557992935181,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.4752926826477051,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.7835660576820374,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4945652961730957,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.689082682132721,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4578232765197754,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.8787635564804077,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.5014208316802978,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5683997273445129,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.4341254234313965,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.8870652914047241,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5459663391113281,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 1.156484842300415,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.4926918983459473,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.7714535593986511,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.53131103515625,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7258591651916504,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.45943522453308105,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.2155957221984863,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.6043071746826172,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6804623007774353,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.4359785556793213,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.45761024951934814,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3847378730773926,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.8509073853492737,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4396988391876221,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.8098800778388977,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.4133442401885986,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.7294316291809082,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.47240419387817384,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.7199881076812744,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40730462074279783,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.5415948629379272,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.36444683074951173,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 1.285866618156433,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.42740211486816404,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6586265563964844,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4160896301269531,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.6770569086074829,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5104285717010498,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.640552818775177,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5205618858337402,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.9050871729850769,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.4352677345275879,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.756203293800354,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.447373104095459,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6095989942550659,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.5091766357421875,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6111555099487305,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.41835684776306153,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.7675306797027588,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.47201828956604003,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6159079670906067,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5127533912658692,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5745006203651428,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48856701850891116,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6177580952644348,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39819822311401365,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7953127026557922,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.4401564598083496,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 1.065096378326416,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.40187416076660154,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6611851453781128,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4557036399841309,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7707319855690002,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.4707478046417236,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3965870440006256,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.41887879371643066,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.6925886869430542,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39414989948272705,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4225500822067261,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46974954605102537,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5246486663818359,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4359858512878418,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.7797112464904785,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4812413215637207,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.7133896946907043,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.4543005466461182,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.8236000537872314,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48366827964782716,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.6479238867759705,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4138167381286621,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.7929301857948303,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.48261222839355467,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.362284779548645,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.534283971786499,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.658808708190918,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.41163101196289065,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.516070544719696,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.4010306358337402,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6887349486351013,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40149264335632323,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.5056571960449219,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3919940710067749,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.0694760084152222,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4507146835327148,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6502102017402649,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.4376358985900879,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.5950688719749451,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4519078254699707,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5176505446434021,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.4313666820526123,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3828600347042084,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.44180989265441895,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6753257513046265,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5487648963928222,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6115583181381226,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4736178398132324,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.40781980752944946,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.3815795421600342,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6663995385169983,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.46039581298828125,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5568227171897888,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42682709693908694,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5221118927001953,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.396131706237793,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8339316844940186,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4795412540435791,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6504913568496704,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.4785301208496094,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6242275238037109,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4487310409545898,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6439151167869568,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.4335296630859375,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6024753451347351,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4808655738830566,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.686862587928772,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4313629627227783,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7613002061843872,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4753777503967285,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7629942297935486,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.4324016571044922,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8611165285110474,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46132574081420896,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.43353787064552307,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.373333215713501,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.49695688486099243,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.48189077377319334,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.515783965587616,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.4596214771270752,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.6876562237739563,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.35883824825286864,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.42947614192962646,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42058372497558594,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4379066824913025,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3526319026947021,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.5771088600158691,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.37990965843200686,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.3974665403366089,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6483776569366455,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9839111566543579,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.40949182510375975,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4381893575191498,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3897416353225708,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.350216269493103,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.43353681564331054,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.48635414242744446,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.40407662391662597,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.7840670943260193,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.4466562271118164,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6072182655334473,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.396435022354126,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.5676800012588501,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.37458391189575196,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.7189476490020752,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4634994983673096,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.42080748081207275,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43068742752075195,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.7519166469573975,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5017862319946289,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6198945045471191,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4238149166107178,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3814507722854614,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4124739170074463,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.39002591371536255,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4085509777069092,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.6172817349433899,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4789612293243408,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.6391016244888306,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.4075304985046387,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.594118595123291,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4542539596557617,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9341249465942383,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42259893417358396,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.7288709282875061,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.42269001007080076,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.5448115468025208,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45872178077697756,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5901434421539307,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7925439834594726,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43836620450019836,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4136178970336914,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.49646082520484924,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.3827540159225464,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5691694021224976,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4286805152893066,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.7288464307785034,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3853081226348877,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 1.2016069889068604,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.402403450012207,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.5269529819488525,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.41291465759277346,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.7392662167549133,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.4353643417358398,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.8992120027542114,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.45477585792541503,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.626833975315094,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43680391311645506,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.49689677357673645,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4409040451049805,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6282675266265869,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42667112350463865,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.7126922607421875,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45916152000427246,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5580125451087952,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4701026439666748,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.43736886978149414,
+      "learning_rate": 5e-05,
+      "loss": 0.3640265941619873,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.8759464621543884,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44643449783325195,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4398824870586395,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.3691298484802246,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.562538206577301,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.47481765747070315,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.4328842759132385,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.4613480567932129,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5684780478477478,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.42593884468078613,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.519466757774353,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4044198513031006,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.8425558805465698,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42935881614685056,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.7469388842582703,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35210604667663575,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5494261980056763,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36253814697265624,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.4391236901283264,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4758582592010498,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.7353135347366333,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3888124942779541,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.6122128367424011,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.3988348960876465,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8968481421470642,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.4200006008148193,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5183577537536621,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43859076499938965,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5962955951690674,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.3618325710296631,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.6844689249992371,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.419573450088501,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 2.3041207790374756,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43375449180603026,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.3782942593097687,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.34841461181640626,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5362056493759155,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.4388935089111328,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.8679549694061279,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48665695190429686,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.38520702719688416,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31835892200469973,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5709502696990967,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.4383851051330566,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.513369083404541,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.41875276565551756,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.5494701862335205,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7724744319915772,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.4949307143688202,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.39315123558044435,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.32247358560562134,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.40227518081665037,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5881837606430054,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4585836887359619,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.0115292072296143,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.36199212074279785,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9708449244499207,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5711725234985352,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.5810522437095642,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3762980937957764,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.9754661321640015,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43689351081848143,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.7655085325241089,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554137945175171,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6773413419723511,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.45856328010559083,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.4474434852600098,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.4327108860015869,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4549546241760254,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4396686553955078,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5432566404342651,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.3287363529205322,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.6105192303657532,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3461976766586304,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5887757539749146,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.4058669090270996,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3826632797718048,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.4288635730743408,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7107186913490295,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38454523086547854,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.6407152414321899,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.3689358472824097,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5461570620536804,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.38851041793823243,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6236541271209717,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40999202728271483,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4176172614097595,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.3426840305328369,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.44086140394210815,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4143134593963623,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.6073216199874878,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.46999621391296387,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5244797468185425,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.39008255004882814,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6138545274734497,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.43625812530517577,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5565832257270813,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39960579872131347,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.49259600043296814,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3722024440765381,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.546994686126709,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3682907819747925,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.478095144033432,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.37905890941619874,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5770573616027832,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.45574336051940917,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5589016675949097,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.3476081609725952,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6209889054298401,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.3975348949432373,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.428181529045105,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.37008640766143797,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.5961155295372009,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.6844874858856201,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.638836681842804,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3731102228164673,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.7055111527442932,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7501702308654785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.610455334186554,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024374008178711,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.612060546875,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4162275791168213,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.468790203332901,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.3881744384765625,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.7208874225616455,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.39170546531677247,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6513557434082031,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.4391339302062988,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.45648840069770813,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39624121189117434,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.697978675365448,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4661447525024414,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.753243088722229,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.394376540184021,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.7163575887680054,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4060837268829346,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.7665302753448486,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.432415246963501,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6077792048454285,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.3501024007797241,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.882154643535614,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47015953063964844,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.423498272895813,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.387217116355896,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.5810433030128479,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44982328414916994,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.39362671971321106,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3725650548934937,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.41626113653182983,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44669065475463865,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 1.006791353225708,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4457366466522217,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.502176821231842,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.42588119506835936,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4030306935310364,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.4012446403503418,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.573884904384613,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.44045190811157225,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.4829924404621124,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.4151927947998047,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4790762960910797,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4421385288238525,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.36565345525741577,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.41750688552856446,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.724289059638977,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4673739433288574,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.9092391729354858,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.3644559860229492,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.679870069026947,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.4468817710876465,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.737008810043335,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.35079207420349123,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.597590446472168,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.40719895362854003,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5171096920967102,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42395763397216796,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.9238699078559875,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.38481290340423585,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5502796173095703,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.40758719444274905,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5856905579566956,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41455903053283694,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.4448857605457306,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3878758907318115,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.42990583181381226,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3641160011291504,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6104210019111633,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.4024786472320557,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8527662754058838,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.452549409866333,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6835286021232605,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42240447998046876,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6754205226898193,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.38313636779785154,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 1.0334895849227905,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42669315338134767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5506278872489929,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.36047968864440916,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5626630187034607,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.41873970031738283,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5406614542007446,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.3593956708908081,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.605750560760498,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.3540114641189575,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.5964953899383545,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.4429689884185791,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6756887435913086,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36358070373535156,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.49958691000938416,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.3598761081695557,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.42558741569519043,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.39934353828430175,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 1.0632734298706055,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.440138053894043,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.1033124923706055,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.3676408052444458,
+      "step": 1205
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5323923576872166e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ae80f32e580d17d750fe77982942ba76a97afe1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/config.json
@@ -0,0 +1,260 @@
+{
+  "adaptor_input_dim": 4096,
+  "architectures": [
+    "MossSpeakerRopeForConditionalGeneration"
+  ],
+  "audio_config": {
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50256
+    ],
+    "bos_token_id": 50256,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 6,
+    "decoder_ffn_dim": 1536,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": 50257,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50256,
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_mel_bins": 80,
+    "pad_token_id": 50256,
+    "scale_embedding": false,
+    "suppress_tokens": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_merge_size": 4,
+  "audio_token_id": 151671,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "model_type": "moss_speaker_rope",
+  "pad_token_id": 151643,
+  "speaker_embed_dim": 192,
+  "speaker_encoder_args": {
+    "C": 64,
+    "F": 72,
+    "block_1d_type": "conv+att",
+    "block_2d_type": "basic_resnet",
+    "causal": "none",
+    "compress_tconvs": true,
+    "emb_bn": false,
+    "embed_dim": 192,
+    "feat_dim": 72,
+    "fm_weigthing_type": "NC",
+    "global_context_att": true,
+    "group_divisor": 1,
+    "hop_length": 160,
+    "out_channels": 224,
+    "pooling_func": "ASTP",
+    "return_2d_output": true,
+    "spec": null,
+    "spec_in_channels": 1,
+    "stages_setup": [
+      [
+        [
+          1,
+          1
+        ],
+        3,
+        3,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        4,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        64
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        5,
+        2,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        5,
+        1,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        48
+      ],
+      [
+        [
+          1,
+          2
+        ],
+        4,
+        0.75,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        32
+      ],
+      [
+        [
+          2,
+          1
+        ],
+        3,
+        0.5,
+        [
+          [
+            3,
+            3
+          ]
+        ],
+        24
+      ]
+    ]
+  },
+  "speaker_plane_indices": [
+    3,
+    11,
+    19,
+    27,
+    35,
+    43,
+    51,
+    59
+  ],
+  "speaker_plane_order": "interleaved",
+  "speaker_rotary_dim": 16,
+  "speaker_rotary_inference_seed": 42,
+  "speaker_rotary_std": 0.02,
+  "speaker_sample_rate": 16000,
+  "speaker_tfmel_args": {
+    "do_preemph": true,
+    "do_spec_aug": false,
+    "eps": 1e-08,
+    "f_max": 7600,
+    "f_min": 20,
+    "hop_length": 160,
+    "n_fft": 512,
+    "n_mels": 72,
+    "norm_signal": true,
+    "win_length": 400
+  },
+  "speaker_window_seconds": 2.0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.0",
+  "use_cache": false
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/generation_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..02ecdacde062a1ffe3a7c8c893ce77674a9c0635
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "max_new_tokens": 5120,
+  "pad_token_id": 151643,
+  "transformers_version": "5.14.0"
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/model.safetensors b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..00b363199c216683c07d05d864031c91c14c8fdd
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f597a440f95071e43cd3566ef23b212475c9e20fb9fe7a9793a27b724638601
+size 1868024544
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/processor_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/processor_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ea20778938d780752c6a5d11bdbcbe3d44df469
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/processor_config.json
@@ -0,0 +1,22 @@
+{
+  "audio_merge_size": 4,
+  "audio_tokens_per_second": 12.5,
+  "enable_time_marker": true,
+  "feature_extractor": {
+    "chunk_length": 30,
+    "dither": 0.0,
+    "feature_extractor_type": "WhisperFeatureExtractor",
+    "feature_size": 80,
+    "hop_length": 160,
+    "n_fft": 400,
+    "n_samples": 480000,
+    "nb_max_frames": 3000,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  },
+  "processor_class": "MossSpeakerRopeProcessor",
+  "speaker_window_seconds": 2.0,
+  "time_marker_every_seconds": 5
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..8f3fe62e4fcd17442942325dec95aa58e731e4a7
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb21895752d2380b32ae2e2c40905c7aa6c8e55c5e6ba33cd065bf77d4d53f74
+size 11423344
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer_config.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d864dbeb3e5f3acf116f6be5de4e77f3735c6734
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/tokenizer_config.json
@@ -0,0 +1,22 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|audio_pad|>"
+  ],
+  "fix_mistral_regex": true,
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "MossSpeakerRopeProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/train_results.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/train_results.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ed41ef7c778cbd27edc07726367ac63287f0d51
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/train_results.json
@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 2.5323923576872166e+17,
+    "train_loss": 0.4422070802295119,
+    "train_runtime": 20343.89,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.059
+}
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/trainer_state.json b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6540974af82b597952b999d8fa935fa38cac6e2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/trainer_state.json
@@ -0,0 +1,1730 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1207,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004142502071251036,
+      "grad_norm": 31.01896095275879,
+      "learning_rate": 3.3057851239669424e-06,
+      "loss": 1.1710086822509767,
+      "step": 5
+    },
+    {
+      "epoch": 0.008285004142502071,
+      "grad_norm": 38.59324264526367,
+      "learning_rate": 7.43801652892562e-06,
+      "loss": 0.7715087890625,
+      "step": 10
+    },
+    {
+      "epoch": 0.012427506213753107,
+      "grad_norm": 7.205821514129639,
+      "learning_rate": 1.1570247933884299e-05,
+      "loss": 0.7687691211700439,
+      "step": 15
+    },
+    {
+      "epoch": 0.016570008285004142,
+      "grad_norm": 2.734048843383789,
+      "learning_rate": 1.5702479338842978e-05,
+      "loss": 0.49406042098999026,
+      "step": 20
+    },
+    {
+      "epoch": 0.020712510356255178,
+      "grad_norm": 3.970097780227661,
+      "learning_rate": 1.9834710743801653e-05,
+      "loss": 0.5342318058013916,
+      "step": 25
+    },
+    {
+      "epoch": 0.024855012427506214,
+      "grad_norm": 3.3853213787078857,
+      "learning_rate": 2.396694214876033e-05,
+      "loss": 0.5030611991882324,
+      "step": 30
+    },
+    {
+      "epoch": 0.02899751449875725,
+      "grad_norm": 1.3471293449401855,
+      "learning_rate": 2.809917355371901e-05,
+      "loss": 0.5010397434234619,
+      "step": 35
+    },
+    {
+      "epoch": 0.033140016570008285,
+      "grad_norm": 1.2355350255966187,
+      "learning_rate": 3.2231404958677685e-05,
+      "loss": 0.5080785274505615,
+      "step": 40
+    },
+    {
+      "epoch": 0.037282518641259324,
+      "grad_norm": 1.4255725145339966,
+      "learning_rate": 3.6363636363636364e-05,
+      "loss": 0.4955380916595459,
+      "step": 45
+    },
+    {
+      "epoch": 0.041425020712510356,
+      "grad_norm": 1.175504207611084,
+      "learning_rate": 4.049586776859504e-05,
+      "loss": 0.4213558673858643,
+      "step": 50
+    },
+    {
+      "epoch": 0.045567522783761395,
+      "grad_norm": 1.0350234508514404,
+      "learning_rate": 4.462809917355372e-05,
+      "loss": 0.4653134346008301,
+      "step": 55
+    },
+    {
+      "epoch": 0.04971002485501243,
+      "grad_norm": 0.8828385472297668,
+      "learning_rate": 4.87603305785124e-05,
+      "loss": 0.46036972999572756,
+      "step": 60
+    },
+    {
+      "epoch": 0.053852526926263466,
+      "grad_norm": 0.5872329473495483,
+      "learning_rate": 5.289256198347108e-05,
+      "loss": 0.4022225379943848,
+      "step": 65
+    },
+    {
+      "epoch": 0.0579950289975145,
+      "grad_norm": 0.9258399605751038,
+      "learning_rate": 5.702479338842975e-05,
+      "loss": 0.37816824913024905,
+      "step": 70
+    },
+    {
+      "epoch": 0.06213753106876554,
+      "grad_norm": 1.0011563301086426,
+      "learning_rate": 6.115702479338842e-05,
+      "loss": 0.40845155715942383,
+      "step": 75
+    },
+    {
+      "epoch": 0.06628003314001657,
+      "grad_norm": 1.2110610008239746,
+      "learning_rate": 6.528925619834711e-05,
+      "loss": 0.46489362716674804,
+      "step": 80
+    },
+    {
+      "epoch": 0.07042253521126761,
+      "grad_norm": 0.9312787652015686,
+      "learning_rate": 6.94214876033058e-05,
+      "loss": 0.4639500617980957,
+      "step": 85
+    },
+    {
+      "epoch": 0.07456503728251865,
+      "grad_norm": 0.6982877850532532,
+      "learning_rate": 7.355371900826447e-05,
+      "loss": 0.7971051692962646,
+      "step": 90
+    },
+    {
+      "epoch": 0.07870753935376967,
+      "grad_norm": 1.2401320934295654,
+      "learning_rate": 7.768595041322314e-05,
+      "loss": 0.48281636238098147,
+      "step": 95
+    },
+    {
+      "epoch": 0.08285004142502071,
+      "grad_norm": 1.1120147705078125,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 0.5136648178100586,
+      "step": 100
+    },
+    {
+      "epoch": 0.08699254349627175,
+      "grad_norm": 0.9892898797988892,
+      "learning_rate": 8.595041322314051e-05,
+      "loss": 0.4707803249359131,
+      "step": 105
+    },
+    {
+      "epoch": 0.09113504556752279,
+      "grad_norm": 0.6651239991188049,
+      "learning_rate": 9.008264462809917e-05,
+      "loss": 0.48854589462280273,
+      "step": 110
+    },
+    {
+      "epoch": 0.09527754763877382,
+      "grad_norm": 0.8432604670524597,
+      "learning_rate": 9.421487603305785e-05,
+      "loss": 0.46013813018798827,
+      "step": 115
+    },
+    {
+      "epoch": 0.09942004971002485,
+      "grad_norm": 0.9123105406761169,
+      "learning_rate": 9.834710743801654e-05,
+      "loss": 0.4647829055786133,
+      "step": 120
+    },
+    {
+      "epoch": 0.1035625517812759,
+      "grad_norm": 0.8377557992935181,
+      "learning_rate": 9.999811713194481e-05,
+      "loss": 0.4752926826477051,
+      "step": 125
+    },
+    {
+      "epoch": 0.10770505385252693,
+      "grad_norm": 0.7835660576820374,
+      "learning_rate": 9.99866112295902e-05,
+      "loss": 0.4945652961730957,
+      "step": 130
+    },
+    {
+      "epoch": 0.11184755592377796,
+      "grad_norm": 0.689082682132721,
+      "learning_rate": 9.996464786686644e-05,
+      "loss": 0.4578232765197754,
+      "step": 135
+    },
+    {
+      "epoch": 0.115990057995029,
+      "grad_norm": 0.8787635564804077,
+      "learning_rate": 9.993223163862385e-05,
+      "loss": 0.5014208316802978,
+      "step": 140
+    },
+    {
+      "epoch": 0.12013256006628004,
+      "grad_norm": 0.5683997273445129,
+      "learning_rate": 9.988936932650681e-05,
+      "loss": 0.4341254234313965,
+      "step": 145
+    },
+    {
+      "epoch": 0.12427506213753108,
+      "grad_norm": 0.8870652914047241,
+      "learning_rate": 9.983606989753509e-05,
+      "loss": 0.5459663391113281,
+      "step": 150
+    },
+    {
+      "epoch": 0.12841756420878211,
+      "grad_norm": 1.156484842300415,
+      "learning_rate": 9.977234450222783e-05,
+      "loss": 0.4926918983459473,
+      "step": 155
+    },
+    {
+      "epoch": 0.13256006628003314,
+      "grad_norm": 0.7714535593986511,
+      "learning_rate": 9.969820647227086e-05,
+      "loss": 0.53131103515625,
+      "step": 160
+    },
+    {
+      "epoch": 0.13670256835128416,
+      "grad_norm": 0.7258591651916504,
+      "learning_rate": 9.961367131772759e-05,
+      "loss": 0.45943522453308105,
+      "step": 165
+    },
+    {
+      "epoch": 0.14084507042253522,
+      "grad_norm": 3.2155957221984863,
+      "learning_rate": 9.951875672379424e-05,
+      "loss": 0.6043071746826172,
+      "step": 170
+    },
+    {
+      "epoch": 0.14498757249378624,
+      "grad_norm": 0.6804623007774353,
+      "learning_rate": 9.941348254710003e-05,
+      "loss": 0.4359785556793213,
+      "step": 175
+    },
+    {
+      "epoch": 0.1491300745650373,
+      "grad_norm": 0.45761024951934814,
+      "learning_rate": 9.929787081155304e-05,
+      "loss": 0.3847378730773926,
+      "step": 180
+    },
+    {
+      "epoch": 0.15327257663628832,
+      "grad_norm": 0.8509073853492737,
+      "learning_rate": 9.91719457037327e-05,
+      "loss": 0.4396988391876221,
+      "step": 185
+    },
+    {
+      "epoch": 0.15741507870753935,
+      "grad_norm": 0.8098800778388977,
+      "learning_rate": 9.903573356782977e-05,
+      "loss": 0.4133442401885986,
+      "step": 190
+    },
+    {
+      "epoch": 0.1615575807787904,
+      "grad_norm": 0.7294316291809082,
+      "learning_rate": 9.888926290013521e-05,
+      "loss": 0.47240419387817384,
+      "step": 195
+    },
+    {
+      "epoch": 0.16570008285004142,
+      "grad_norm": 0.7199881076812744,
+      "learning_rate": 9.873256434307828e-05,
+      "loss": 0.40730462074279783,
+      "step": 200
+    },
+    {
+      "epoch": 0.16984258492129245,
+      "grad_norm": 0.5415948629379272,
+      "learning_rate": 9.856567067881628e-05,
+      "loss": 0.36444683074951173,
+      "step": 205
+    },
+    {
+      "epoch": 0.1739850869925435,
+      "grad_norm": 1.285866618156433,
+      "learning_rate": 9.838861682237618e-05,
+      "loss": 0.42740211486816404,
+      "step": 210
+    },
+    {
+      "epoch": 0.17812758906379453,
+      "grad_norm": 0.6586265563964844,
+      "learning_rate": 9.820143981435031e-05,
+      "loss": 0.4160896301269531,
+      "step": 215
+    },
+    {
+      "epoch": 0.18227009113504558,
+      "grad_norm": 0.6770569086074829,
+      "learning_rate": 9.800417881314716e-05,
+      "loss": 0.5104285717010498,
+      "step": 220
+    },
+    {
+      "epoch": 0.1864125932062966,
+      "grad_norm": 0.640552818775177,
+      "learning_rate": 9.779687508679935e-05,
+      "loss": 0.5205618858337402,
+      "step": 225
+    },
+    {
+      "epoch": 0.19055509527754763,
+      "grad_norm": 0.9050871729850769,
+      "learning_rate": 9.75795720043301e-05,
+      "loss": 0.4352677345275879,
+      "step": 230
+    },
+    {
+      "epoch": 0.19469759734879868,
+      "grad_norm": 0.756203293800354,
+      "learning_rate": 9.73523150266802e-05,
+      "loss": 0.447373104095459,
+      "step": 235
+    },
+    {
+      "epoch": 0.1988400994200497,
+      "grad_norm": 0.6095989942550659,
+      "learning_rate": 9.711515169719732e-05,
+      "loss": 0.5091766357421875,
+      "step": 240
+    },
+    {
+      "epoch": 0.20298260149130073,
+      "grad_norm": 0.6111555099487305,
+      "learning_rate": 9.686813163168973e-05,
+      "loss": 0.41835684776306153,
+      "step": 245
+    },
+    {
+      "epoch": 0.2071251035625518,
+      "grad_norm": 0.7675306797027588,
+      "learning_rate": 9.661130650804643e-05,
+      "loss": 0.47201828956604003,
+      "step": 250
+    },
+    {
+      "epoch": 0.2112676056338028,
+      "grad_norm": 0.6159079670906067,
+      "learning_rate": 9.634473005542579e-05,
+      "loss": 0.5127533912658692,
+      "step": 255
+    },
+    {
+      "epoch": 0.21541010770505387,
+      "grad_norm": 0.5745006203651428,
+      "learning_rate": 9.606845804301524e-05,
+      "loss": 0.48856701850891116,
+      "step": 260
+    },
+    {
+      "epoch": 0.2195526097763049,
+      "grad_norm": 0.6177580952644348,
+      "learning_rate": 9.578254826836391e-05,
+      "loss": 0.39819822311401365,
+      "step": 265
+    },
+    {
+      "epoch": 0.22369511184755592,
+      "grad_norm": 0.7953127026557922,
+      "learning_rate": 9.548706054529124e-05,
+      "loss": 0.4401564598083496,
+      "step": 270
+    },
+    {
+      "epoch": 0.22783761391880697,
+      "grad_norm": 1.065096378326416,
+      "learning_rate": 9.518205669137352e-05,
+      "loss": 0.40187416076660154,
+      "step": 275
+    },
+    {
+      "epoch": 0.231980115990058,
+      "grad_norm": 0.6611851453781128,
+      "learning_rate": 9.486760051501139e-05,
+      "loss": 0.4557036399841309,
+      "step": 280
+    },
+    {
+      "epoch": 0.23612261806130902,
+      "grad_norm": 0.7707319855690002,
+      "learning_rate": 9.454375780208071e-05,
+      "loss": 0.4707478046417236,
+      "step": 285
+    },
+    {
+      "epoch": 0.24026512013256007,
+      "grad_norm": 0.3965870440006256,
+      "learning_rate": 9.421059630216991e-05,
+      "loss": 0.41887879371643066,
+      "step": 290
+    },
+    {
+      "epoch": 0.2444076222038111,
+      "grad_norm": 0.6925886869430542,
+      "learning_rate": 9.38681857144063e-05,
+      "loss": 0.39414989948272705,
+      "step": 295
+    },
+    {
+      "epoch": 0.24855012427506215,
+      "grad_norm": 0.4225500822067261,
+      "learning_rate": 9.351659767287478e-05,
+      "loss": 0.46974954605102537,
+      "step": 300
+    },
+    {
+      "epoch": 0.2526926263463132,
+      "grad_norm": 0.5246486663818359,
+      "learning_rate": 9.315590573163153e-05,
+      "loss": 0.4359858512878418,
+      "step": 305
+    },
+    {
+      "epoch": 0.25683512841756423,
+      "grad_norm": 0.7797112464904785,
+      "learning_rate": 9.278618534931611e-05,
+      "loss": 0.4812413215637207,
+      "step": 310
+    },
+    {
+      "epoch": 0.2609776304888152,
+      "grad_norm": 0.7133896946907043,
+      "learning_rate": 9.240751387336522e-05,
+      "loss": 0.4543005466461182,
+      "step": 315
+    },
+    {
+      "epoch": 0.2651201325600663,
+      "grad_norm": 0.8236000537872314,
+      "learning_rate": 9.201997052383106e-05,
+      "loss": 0.48366827964782716,
+      "step": 320
+    },
+    {
+      "epoch": 0.26926263463131733,
+      "grad_norm": 0.6479238867759705,
+      "learning_rate": 9.162363637680822e-05,
+      "loss": 0.4138167381286621,
+      "step": 325
+    },
+    {
+      "epoch": 0.27340513670256833,
+      "grad_norm": 0.7929301857948303,
+      "learning_rate": 9.121859434747204e-05,
+      "loss": 0.48261222839355467,
+      "step": 330
+    },
+    {
+      "epoch": 0.2775476387738194,
+      "grad_norm": 1.362284779548645,
+      "learning_rate": 9.080492917273237e-05,
+      "loss": 0.534283971786499,
+      "step": 335
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 0.658808708190918,
+      "learning_rate": 9.038272739350618e-05,
+      "loss": 0.41163101196289065,
+      "step": 340
+    },
+    {
+      "epoch": 0.28583264291632143,
+      "grad_norm": 0.516070544719696,
+      "learning_rate": 8.995207733661267e-05,
+      "loss": 0.4010306358337402,
+      "step": 345
+    },
+    {
+      "epoch": 0.2899751449875725,
+      "grad_norm": 0.6887349486351013,
+      "learning_rate": 8.951306909629492e-05,
+      "loss": 0.40149264335632323,
+      "step": 350
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 0.5056571960449219,
+      "learning_rate": 8.906579451537171e-05,
+      "loss": 0.3919940710067749,
+      "step": 355
+    },
+    {
+      "epoch": 0.2982601491300746,
+      "grad_norm": 1.0694760084152222,
+      "learning_rate": 8.861034716602343e-05,
+      "loss": 0.4507146835327148,
+      "step": 360
+    },
+    {
+      "epoch": 0.3024026512013256,
+      "grad_norm": 0.6502102017402649,
+      "learning_rate": 8.814682233021636e-05,
+      "loss": 0.4376358985900879,
+      "step": 365
+    },
+    {
+      "epoch": 0.30654515327257664,
+      "grad_norm": 0.5950688719749451,
+      "learning_rate": 8.76753169797692e-05,
+      "loss": 0.4519078254699707,
+      "step": 370
+    },
+    {
+      "epoch": 0.3106876553438277,
+      "grad_norm": 0.5176505446434021,
+      "learning_rate": 8.719592975606601e-05,
+      "loss": 0.4313666820526123,
+      "step": 375
+    },
+    {
+      "epoch": 0.3148301574150787,
+      "grad_norm": 0.3828600347042084,
+      "learning_rate": 8.670876094941991e-05,
+      "loss": 0.44180989265441895,
+      "step": 380
+    },
+    {
+      "epoch": 0.31897265948632975,
+      "grad_norm": 0.6753257513046265,
+      "learning_rate": 8.62139124780919e-05,
+      "loss": 0.5487648963928222,
+      "step": 385
+    },
+    {
+      "epoch": 0.3231151615575808,
+      "grad_norm": 0.6115583181381226,
+      "learning_rate": 8.571148786696894e-05,
+      "loss": 0.4736178398132324,
+      "step": 390
+    },
+    {
+      "epoch": 0.3272576636288318,
+      "grad_norm": 0.40781980752944946,
+      "learning_rate": 8.520159222590605e-05,
+      "loss": 0.3815795421600342,
+      "step": 395
+    },
+    {
+      "epoch": 0.33140016570008285,
+      "grad_norm": 0.6663995385169983,
+      "learning_rate": 8.468433222773677e-05,
+      "loss": 0.46039581298828125,
+      "step": 400
+    },
+    {
+      "epoch": 0.3355426677713339,
+      "grad_norm": 0.5568227171897888,
+      "learning_rate": 8.415981608595667e-05,
+      "loss": 0.42682709693908694,
+      "step": 405
+    },
+    {
+      "epoch": 0.3396851698425849,
+      "grad_norm": 0.5221118927001953,
+      "learning_rate": 8.36281535320844e-05,
+      "loss": 0.396131706237793,
+      "step": 410
+    },
+    {
+      "epoch": 0.34382767191383595,
+      "grad_norm": 0.8339316844940186,
+      "learning_rate": 8.30894557927055e-05,
+      "loss": 0.4795412540435791,
+      "step": 415
+    },
+    {
+      "epoch": 0.347970173985087,
+      "grad_norm": 0.6504913568496704,
+      "learning_rate": 8.254383556620299e-05,
+      "loss": 0.4785301208496094,
+      "step": 420
+    },
+    {
+      "epoch": 0.352112676056338,
+      "grad_norm": 0.6242275238037109,
+      "learning_rate": 8.199140699918049e-05,
+      "loss": 0.4487310409545898,
+      "step": 425
+    },
+    {
+      "epoch": 0.35625517812758906,
+      "grad_norm": 0.6439151167869568,
+      "learning_rate": 8.143228566258192e-05,
+      "loss": 0.4335296630859375,
+      "step": 430
+    },
+    {
+      "epoch": 0.3603976801988401,
+      "grad_norm": 0.6024753451347351,
+      "learning_rate": 8.08665885275137e-05,
+      "loss": 0.4808655738830566,
+      "step": 435
+    },
+    {
+      "epoch": 0.36454018227009116,
+      "grad_norm": 0.686862587928772,
+      "learning_rate": 8.029443394077356e-05,
+      "loss": 0.4313629627227783,
+      "step": 440
+    },
+    {
+      "epoch": 0.36868268434134216,
+      "grad_norm": 0.7613002061843872,
+      "learning_rate": 7.971594160009194e-05,
+      "loss": 0.4753777503967285,
+      "step": 445
+    },
+    {
+      "epoch": 0.3728251864125932,
+      "grad_norm": 0.7629942297935486,
+      "learning_rate": 7.913123252909056e-05,
+      "loss": 0.4324016571044922,
+      "step": 450
+    },
+    {
+      "epoch": 0.37696768848384427,
+      "grad_norm": 0.8611165285110474,
+      "learning_rate": 7.854042905196354e-05,
+      "loss": 0.46132574081420896,
+      "step": 455
+    },
+    {
+      "epoch": 0.38111019055509526,
+      "grad_norm": 0.43353787064552307,
+      "learning_rate": 7.794365476788667e-05,
+      "loss": 0.373333215713501,
+      "step": 460
+    },
+    {
+      "epoch": 0.3852526926263463,
+      "grad_norm": 0.49695688486099243,
+      "learning_rate": 7.734103452515975e-05,
+      "loss": 0.48189077377319334,
+      "step": 465
+    },
+    {
+      "epoch": 0.38939519469759737,
+      "grad_norm": 0.515783965587616,
+      "learning_rate": 7.67326943950877e-05,
+      "loss": 0.4596214771270752,
+      "step": 470
+    },
+    {
+      "epoch": 0.39353769676884837,
+      "grad_norm": 0.6876562237739563,
+      "learning_rate": 7.611876164560578e-05,
+      "loss": 0.35883824825286864,
+      "step": 475
+    },
+    {
+      "epoch": 0.3976801988400994,
+      "grad_norm": 0.42947614192962646,
+      "learning_rate": 7.549936471465444e-05,
+      "loss": 0.42058372497558594,
+      "step": 480
+    },
+    {
+      "epoch": 0.40182270091135047,
+      "grad_norm": 0.4379066824913025,
+      "learning_rate": 7.487463318330946e-05,
+      "loss": 0.3526319026947021,
+      "step": 485
+    },
+    {
+      "epoch": 0.40596520298260147,
+      "grad_norm": 0.5771088600158691,
+      "learning_rate": 7.42446977486729e-05,
+      "loss": 0.37990965843200686,
+      "step": 490
+    },
+    {
+      "epoch": 0.4101077050538525,
+      "grad_norm": 1.3974665403366089,
+      "learning_rate": 7.360969019653067e-05,
+      "loss": 0.6483776569366455,
+      "step": 495
+    },
+    {
+      "epoch": 0.4142502071251036,
+      "grad_norm": 0.9839111566543579,
+      "learning_rate": 7.296974337378209e-05,
+      "loss": 0.40949182510375975,
+      "step": 500
+    },
+    {
+      "epoch": 0.4183927091963546,
+      "grad_norm": 0.4381893575191498,
+      "learning_rate": 7.23249911606477e-05,
+      "loss": 0.3897416353225708,
+      "step": 505
+    },
+    {
+      "epoch": 0.4225352112676056,
+      "grad_norm": 1.350216269493103,
+      "learning_rate": 7.167556844266091e-05,
+      "loss": 0.43353681564331054,
+      "step": 510
+    },
+    {
+      "epoch": 0.4266777133388567,
+      "grad_norm": 0.48635414242744446,
+      "learning_rate": 7.102161108244907e-05,
+      "loss": 0.40407662391662597,
+      "step": 515
+    },
+    {
+      "epoch": 0.43082021541010773,
+      "grad_norm": 0.7840670943260193,
+      "learning_rate": 7.036325589131048e-05,
+      "loss": 0.4466562271118164,
+      "step": 520
+    },
+    {
+      "epoch": 0.43496271748135873,
+      "grad_norm": 0.6072182655334473,
+      "learning_rate": 6.970064060059258e-05,
+      "loss": 0.396435022354126,
+      "step": 525
+    },
+    {
+      "epoch": 0.4391052195526098,
+      "grad_norm": 0.5676800012588501,
+      "learning_rate": 6.903390383287794e-05,
+      "loss": 0.37458391189575196,
+      "step": 530
+    },
+    {
+      "epoch": 0.44324772162386084,
+      "grad_norm": 0.7189476490020752,
+      "learning_rate": 6.83631850729837e-05,
+      "loss": 0.4634994983673096,
+      "step": 535
+    },
+    {
+      "epoch": 0.44739022369511183,
+      "grad_norm": 0.42080748081207275,
+      "learning_rate": 6.768862463878047e-05,
+      "loss": 0.43068742752075195,
+      "step": 540
+    },
+    {
+      "epoch": 0.4515327257663629,
+      "grad_norm": 0.7519166469573975,
+      "learning_rate": 6.701036365183737e-05,
+      "loss": 0.5017862319946289,
+      "step": 545
+    },
+    {
+      "epoch": 0.45567522783761394,
+      "grad_norm": 0.6198945045471191,
+      "learning_rate": 6.632854400789835e-05,
+      "loss": 0.4238149166107178,
+      "step": 550
+    },
+    {
+      "epoch": 0.45981772990886494,
+      "grad_norm": 0.3814507722854614,
+      "learning_rate": 6.564330834719708e-05,
+      "loss": 0.4124739170074463,
+      "step": 555
+    },
+    {
+      "epoch": 0.463960231980116,
+      "grad_norm": 0.39002591371536255,
+      "learning_rate": 6.495480002461577e-05,
+      "loss": 0.4085509777069092,
+      "step": 560
+    },
+    {
+      "epoch": 0.46810273405136704,
+      "grad_norm": 0.6172817349433899,
+      "learning_rate": 6.426316307969465e-05,
+      "loss": 0.4789612293243408,
+      "step": 565
+    },
+    {
+      "epoch": 0.47224523612261804,
+      "grad_norm": 0.6391016244888306,
+      "learning_rate": 6.3568542206498e-05,
+      "loss": 0.4075304985046387,
+      "step": 570
+    },
+    {
+      "epoch": 0.4763877381938691,
+      "grad_norm": 0.594118595123291,
+      "learning_rate": 6.28710827233436e-05,
+      "loss": 0.4542539596557617,
+      "step": 575
+    },
+    {
+      "epoch": 0.48053024026512015,
+      "grad_norm": 0.9341249465942383,
+      "learning_rate": 6.21709305424012e-05,
+      "loss": 0.42259893417358396,
+      "step": 580
+    },
+    {
+      "epoch": 0.48467274233637114,
+      "grad_norm": 0.7288709282875061,
+      "learning_rate": 6.146823213916702e-05,
+      "loss": 0.42269001007080076,
+      "step": 585
+    },
+    {
+      "epoch": 0.4888152444076222,
+      "grad_norm": 0.5448115468025208,
+      "learning_rate": 6.076313452182033e-05,
+      "loss": 0.45872178077697756,
+      "step": 590
+    },
+    {
+      "epoch": 0.49295774647887325,
+      "grad_norm": 0.5901434421539307,
+      "learning_rate": 6.005578520046848e-05,
+      "loss": 0.7925439834594726,
+      "step": 595
+    },
+    {
+      "epoch": 0.4971002485501243,
+      "grad_norm": 0.43836620450019836,
+      "learning_rate": 5.934633215628711e-05,
+      "loss": 0.4136178970336914,
+      "step": 600
+    },
+    {
+      "epoch": 0.5012427506213754,
+      "grad_norm": 0.49646082520484924,
+      "learning_rate": 5.863492381056164e-05,
+      "loss": 0.3827540159225464,
+      "step": 605
+    },
+    {
+      "epoch": 0.5053852526926264,
+      "grad_norm": 0.5691694021224976,
+      "learning_rate": 5.792170899363683e-05,
+      "loss": 0.4286805152893066,
+      "step": 610
+    },
+    {
+      "epoch": 0.5095277547638773,
+      "grad_norm": 0.7288464307785034,
+      "learning_rate": 5.72068369137806e-05,
+      "loss": 0.3853081226348877,
+      "step": 615
+    },
+    {
+      "epoch": 0.5136702568351285,
+      "grad_norm": 1.2016069889068604,
+      "learning_rate": 5.649045712596903e-05,
+      "loss": 0.402403450012207,
+      "step": 620
+    },
+    {
+      "epoch": 0.5178127589063795,
+      "grad_norm": 0.5269529819488525,
+      "learning_rate": 5.577271950059849e-05,
+      "loss": 0.41291465759277346,
+      "step": 625
+    },
+    {
+      "epoch": 0.5219552609776305,
+      "grad_norm": 0.7392662167549133,
+      "learning_rate": 5.505377419213214e-05,
+      "loss": 0.4353643417358398,
+      "step": 630
+    },
+    {
+      "epoch": 0.5260977630488816,
+      "grad_norm": 0.8992120027542114,
+      "learning_rate": 5.433377160768669e-05,
+      "loss": 0.45477585792541503,
+      "step": 635
+    },
+    {
+      "epoch": 0.5302402651201326,
+      "grad_norm": 0.626833975315094,
+      "learning_rate": 5.3612862375566574e-05,
+      "loss": 0.43680391311645506,
+      "step": 640
+    },
+    {
+      "epoch": 0.5343827671913836,
+      "grad_norm": 0.49689677357673645,
+      "learning_rate": 5.2891197313751606e-05,
+      "loss": 0.4409040451049805,
+      "step": 645
+    },
+    {
+      "epoch": 0.5385252692626347,
+      "grad_norm": 0.6282675266265869,
+      "learning_rate": 5.216892739834519e-05,
+      "loss": 0.42667112350463865,
+      "step": 650
+    },
+    {
+      "epoch": 0.5426677713338857,
+      "grad_norm": 0.7126922607421875,
+      "learning_rate": 5.144620373198916e-05,
+      "loss": 0.45916152000427246,
+      "step": 655
+    },
+    {
+      "epoch": 0.5468102734051367,
+      "grad_norm": 0.5580125451087952,
+      "learning_rate": 5.072317751225254e-05,
+      "loss": 0.4701026439666748,
+      "step": 660
+    },
+    {
+      "epoch": 0.5509527754763878,
+      "grad_norm": 0.43736886978149414,
+      "learning_rate": 5e-05,
+      "loss": 0.3640265941619873,
+      "step": 665
+    },
+    {
+      "epoch": 0.5550952775476388,
+      "grad_norm": 0.8759464621543884,
+      "learning_rate": 4.927682248774748e-05,
+      "loss": 0.44643449783325195,
+      "step": 670
+    },
+    {
+      "epoch": 0.5592377796188898,
+      "grad_norm": 0.4398824870586395,
+      "learning_rate": 4.855379626801085e-05,
+      "loss": 0.3691298484802246,
+      "step": 675
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 0.562538206577301,
+      "learning_rate": 4.783107260165483e-05,
+      "loss": 0.47481765747070315,
+      "step": 680
+    },
+    {
+      "epoch": 0.5675227837613919,
+      "grad_norm": 0.4328842759132385,
+      "learning_rate": 4.710880268624841e-05,
+      "loss": 0.4613480567932129,
+      "step": 685
+    },
+    {
+      "epoch": 0.5716652858326429,
+      "grad_norm": 0.5684780478477478,
+      "learning_rate": 4.6387137624433444e-05,
+      "loss": 0.42593884468078613,
+      "step": 690
+    },
+    {
+      "epoch": 0.575807787903894,
+      "grad_norm": 0.519466757774353,
+      "learning_rate": 4.5666228392313315e-05,
+      "loss": 0.4044198513031006,
+      "step": 695
+    },
+    {
+      "epoch": 0.579950289975145,
+      "grad_norm": 0.8425558805465698,
+      "learning_rate": 4.494622580786788e-05,
+      "loss": 0.42935881614685056,
+      "step": 700
+    },
+    {
+      "epoch": 0.584092792046396,
+      "grad_norm": 0.7469388842582703,
+      "learning_rate": 4.422728049940151e-05,
+      "loss": 0.35210604667663575,
+      "step": 705
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.5494261980056763,
+      "learning_rate": 4.3509542874030986e-05,
+      "loss": 0.36253814697265624,
+      "step": 710
+    },
+    {
+      "epoch": 0.5923777961888981,
+      "grad_norm": 0.4391236901283264,
+      "learning_rate": 4.27931630862194e-05,
+      "loss": 0.4758582592010498,
+      "step": 715
+    },
+    {
+      "epoch": 0.5965202982601492,
+      "grad_norm": 1.7353135347366333,
+      "learning_rate": 4.207829100636318e-05,
+      "loss": 0.3888124942779541,
+      "step": 720
+    },
+    {
+      "epoch": 0.6006628003314002,
+      "grad_norm": 0.6122128367424011,
+      "learning_rate": 4.1365076189438365e-05,
+      "loss": 0.3988348960876465,
+      "step": 725
+    },
+    {
+      "epoch": 0.6048053024026512,
+      "grad_norm": 0.8968481421470642,
+      "learning_rate": 4.065366784371289e-05,
+      "loss": 0.4200006008148193,
+      "step": 730
+    },
+    {
+      "epoch": 0.6089478044739023,
+      "grad_norm": 0.5183577537536621,
+      "learning_rate": 3.994421479953153e-05,
+      "loss": 0.43859076499938965,
+      "step": 735
+    },
+    {
+      "epoch": 0.6130903065451533,
+      "grad_norm": 0.5962955951690674,
+      "learning_rate": 3.9236865478179685e-05,
+      "loss": 0.3618325710296631,
+      "step": 740
+    },
+    {
+      "epoch": 0.6172328086164043,
+      "grad_norm": 0.6844689249992371,
+      "learning_rate": 3.853176786083298e-05,
+      "loss": 0.419573450088501,
+      "step": 745
+    },
+    {
+      "epoch": 0.6213753106876554,
+      "grad_norm": 2.3041207790374756,
+      "learning_rate": 3.7829069457598825e-05,
+      "loss": 0.43375449180603026,
+      "step": 750
+    },
+    {
+      "epoch": 0.6255178127589064,
+      "grad_norm": 0.3782942593097687,
+      "learning_rate": 3.7128917276656406e-05,
+      "loss": 0.34841461181640626,
+      "step": 755
+    },
+    {
+      "epoch": 0.6296603148301574,
+      "grad_norm": 0.5362056493759155,
+      "learning_rate": 3.6431457793502e-05,
+      "loss": 0.4388935089111328,
+      "step": 760
+    },
+    {
+      "epoch": 0.6338028169014085,
+      "grad_norm": 0.8679549694061279,
+      "learning_rate": 3.573683692030537e-05,
+      "loss": 0.48665695190429686,
+      "step": 765
+    },
+    {
+      "epoch": 0.6379453189726595,
+      "grad_norm": 0.38520702719688416,
+      "learning_rate": 3.504519997538422e-05,
+      "loss": 0.31835892200469973,
+      "step": 770
+    },
+    {
+      "epoch": 0.6420878210439105,
+      "grad_norm": 0.5709502696990967,
+      "learning_rate": 3.435669165280294e-05,
+      "loss": 0.4383851051330566,
+      "step": 775
+    },
+    {
+      "epoch": 0.6462303231151616,
+      "grad_norm": 0.513369083404541,
+      "learning_rate": 3.3671455992101666e-05,
+      "loss": 0.41875276565551756,
+      "step": 780
+    },
+    {
+      "epoch": 0.6503728251864126,
+      "grad_norm": 0.5494701862335205,
+      "learning_rate": 3.298963634816263e-05,
+      "loss": 0.7724744319915772,
+      "step": 785
+    },
+    {
+      "epoch": 0.6545153272576636,
+      "grad_norm": 0.4949307143688202,
+      "learning_rate": 3.231137536121954e-05,
+      "loss": 0.39315123558044435,
+      "step": 790
+    },
+    {
+      "epoch": 0.6586578293289147,
+      "grad_norm": 0.32247358560562134,
+      "learning_rate": 3.1636814927016324e-05,
+      "loss": 0.40227518081665037,
+      "step": 795
+    },
+    {
+      "epoch": 0.6628003314001657,
+      "grad_norm": 0.5881837606430054,
+      "learning_rate": 3.096609616712207e-05,
+      "loss": 0.4585836887359619,
+      "step": 800
+    },
+    {
+      "epoch": 0.6669428334714167,
+      "grad_norm": 1.0115292072296143,
+      "learning_rate": 3.0299359399407434e-05,
+      "loss": 0.36199212074279785,
+      "step": 805
+    },
+    {
+      "epoch": 0.6710853355426678,
+      "grad_norm": 0.9708449244499207,
+      "learning_rate": 2.963674410868953e-05,
+      "loss": 0.5711725234985352,
+      "step": 810
+    },
+    {
+      "epoch": 0.6752278376139188,
+      "grad_norm": 0.5810522437095642,
+      "learning_rate": 2.8978388917550935e-05,
+      "loss": 0.3762980937957764,
+      "step": 815
+    },
+    {
+      "epoch": 0.6793703396851698,
+      "grad_norm": 0.9754661321640015,
+      "learning_rate": 2.8324431557339105e-05,
+      "loss": 0.43689351081848143,
+      "step": 820
+    },
+    {
+      "epoch": 0.6835128417564209,
+      "grad_norm": 0.7655085325241089,
+      "learning_rate": 2.7675008839352314e-05,
+      "loss": 0.3554137945175171,
+      "step": 825
+    },
+    {
+      "epoch": 0.6876553438276719,
+      "grad_norm": 0.6773413419723511,
+      "learning_rate": 2.703025662621793e-05,
+      "loss": 0.45856328010559083,
+      "step": 830
+    },
+    {
+      "epoch": 0.6917978458989229,
+      "grad_norm": 1.4474434852600098,
+      "learning_rate": 2.6390309803469326e-05,
+      "loss": 0.4327108860015869,
+      "step": 835
+    },
+    {
+      "epoch": 0.695940347970174,
+      "grad_norm": 0.4549546241760254,
+      "learning_rate": 2.5755302251327096e-05,
+      "loss": 0.4396686553955078,
+      "step": 840
+    },
+    {
+      "epoch": 0.700082850041425,
+      "grad_norm": 0.5432566404342651,
+      "learning_rate": 2.5125366816690555e-05,
+      "loss": 0.3287363529205322,
+      "step": 845
+    },
+    {
+      "epoch": 0.704225352112676,
+      "grad_norm": 0.6105192303657532,
+      "learning_rate": 2.450063528534558e-05,
+      "loss": 0.3461976766586304,
+      "step": 850
+    },
+    {
+      "epoch": 0.7083678541839271,
+      "grad_norm": 0.5887757539749146,
+      "learning_rate": 2.3881238354394236e-05,
+      "loss": 0.4058669090270996,
+      "step": 855
+    },
+    {
+      "epoch": 0.7125103562551781,
+      "grad_norm": 0.3826632797718048,
+      "learning_rate": 2.32673056049123e-05,
+      "loss": 0.4288635730743408,
+      "step": 860
+    },
+    {
+      "epoch": 0.7166528583264291,
+      "grad_norm": 0.7107186913490295,
+      "learning_rate": 2.2658965474840262e-05,
+      "loss": 0.38454523086547854,
+      "step": 865
+    },
+    {
+      "epoch": 0.7207953603976802,
+      "grad_norm": 0.6407152414321899,
+      "learning_rate": 2.205634523211335e-05,
+      "loss": 0.3689358472824097,
+      "step": 870
+    },
+    {
+      "epoch": 0.7249378624689312,
+      "grad_norm": 0.5461570620536804,
+      "learning_rate": 2.1459570948036485e-05,
+      "loss": 0.38851041793823243,
+      "step": 875
+    },
+    {
+      "epoch": 0.7290803645401823,
+      "grad_norm": 0.6236541271209717,
+      "learning_rate": 2.086876747090945e-05,
+      "loss": 0.40999202728271483,
+      "step": 880
+    },
+    {
+      "epoch": 0.7332228666114333,
+      "grad_norm": 0.4176172614097595,
+      "learning_rate": 2.0284058399908045e-05,
+      "loss": 0.3426840305328369,
+      "step": 885
+    },
+    {
+      "epoch": 0.7373653686826843,
+      "grad_norm": 0.44086140394210815,
+      "learning_rate": 1.9705566059226448e-05,
+      "loss": 0.4143134593963623,
+      "step": 890
+    },
+    {
+      "epoch": 0.7415078707539354,
+      "grad_norm": 1.6073216199874878,
+      "learning_rate": 1.9133411472486324e-05,
+      "loss": 0.46999621391296387,
+      "step": 895
+    },
+    {
+      "epoch": 0.7456503728251864,
+      "grad_norm": 0.5244797468185425,
+      "learning_rate": 1.8567714337418092e-05,
+      "loss": 0.39008255004882814,
+      "step": 900
+    },
+    {
+      "epoch": 0.7497928748964374,
+      "grad_norm": 0.6138545274734497,
+      "learning_rate": 1.8008593000819518e-05,
+      "loss": 0.43625812530517577,
+      "step": 905
+    },
+    {
+      "epoch": 0.7539353769676885,
+      "grad_norm": 0.5565832257270813,
+      "learning_rate": 1.745616443379699e-05,
+      "loss": 0.39960579872131347,
+      "step": 910
+    },
+    {
+      "epoch": 0.7580778790389395,
+      "grad_norm": 0.49259600043296814,
+      "learning_rate": 1.691054420729452e-05,
+      "loss": 0.3722024440765381,
+      "step": 915
+    },
+    {
+      "epoch": 0.7622203811101905,
+      "grad_norm": 0.546994686126709,
+      "learning_rate": 1.6371846467915603e-05,
+      "loss": 0.3682907819747925,
+      "step": 920
+    },
+    {
+      "epoch": 0.7663628831814416,
+      "grad_norm": 0.478095144033432,
+      "learning_rate": 1.5840183914043338e-05,
+      "loss": 0.37905890941619874,
+      "step": 925
+    },
+    {
+      "epoch": 0.7705053852526926,
+      "grad_norm": 0.5770573616027832,
+      "learning_rate": 1.5315667772263224e-05,
+      "loss": 0.45574336051940917,
+      "step": 930
+    },
+    {
+      "epoch": 0.7746478873239436,
+      "grad_norm": 0.5589016675949097,
+      "learning_rate": 1.4798407774093954e-05,
+      "loss": 0.3476081609725952,
+      "step": 935
+    },
+    {
+      "epoch": 0.7787903893951947,
+      "grad_norm": 0.6209889054298401,
+      "learning_rate": 1.4288512133031073e-05,
+      "loss": 0.3975348949432373,
+      "step": 940
+    },
+    {
+      "epoch": 0.7829328914664457,
+      "grad_norm": 0.428181529045105,
+      "learning_rate": 1.3786087521908103e-05,
+      "loss": 0.37008640766143797,
+      "step": 945
+    },
+    {
+      "epoch": 0.7870753935376967,
+      "grad_norm": 0.5961155295372009,
+      "learning_rate": 1.3291239050580084e-05,
+      "loss": 0.6844874858856201,
+      "step": 950
+    },
+    {
+      "epoch": 0.7912178956089478,
+      "grad_norm": 0.638836681842804,
+      "learning_rate": 1.2804070243933996e-05,
+      "loss": 0.3731102228164673,
+      "step": 955
+    },
+    {
+      "epoch": 0.7953603976801988,
+      "grad_norm": 0.7055111527442932,
+      "learning_rate": 1.2324683020230798e-05,
+      "loss": 0.7501702308654785,
+      "step": 960
+    },
+    {
+      "epoch": 0.7995028997514498,
+      "grad_norm": 0.610455334186554,
+      "learning_rate": 1.1853177669783643e-05,
+      "loss": 0.4024374008178711,
+      "step": 965
+    },
+    {
+      "epoch": 0.8036454018227009,
+      "grad_norm": 0.612060546875,
+      "learning_rate": 1.1389652833976577e-05,
+      "loss": 0.4162275791168213,
+      "step": 970
+    },
+    {
+      "epoch": 0.8077879038939519,
+      "grad_norm": 0.468790203332901,
+      "learning_rate": 1.0934205484628284e-05,
+      "loss": 0.3881744384765625,
+      "step": 975
+    },
+    {
+      "epoch": 0.8119304059652029,
+      "grad_norm": 0.7208874225616455,
+      "learning_rate": 1.0486930903705094e-05,
+      "loss": 0.39170546531677247,
+      "step": 980
+    },
+    {
+      "epoch": 0.816072908036454,
+      "grad_norm": 0.6513557434082031,
+      "learning_rate": 1.0047922663387349e-05,
+      "loss": 0.4391339302062988,
+      "step": 985
+    },
+    {
+      "epoch": 0.820215410107705,
+      "grad_norm": 0.45648840069770813,
+      "learning_rate": 9.617272606493844e-06,
+      "loss": 0.39624121189117434,
+      "step": 990
+    },
+    {
+      "epoch": 0.824357912178956,
+      "grad_norm": 0.697978675365448,
+      "learning_rate": 9.195070827267632e-06,
+      "loss": 0.4661447525024414,
+      "step": 995
+    },
+    {
+      "epoch": 0.8285004142502072,
+      "grad_norm": 0.753243088722229,
+      "learning_rate": 8.781405652527974e-06,
+      "loss": 0.394376540184021,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8326429163214581,
+      "grad_norm": 0.7163575887680054,
+      "learning_rate": 8.376363623191791e-06,
+      "loss": 0.4060837268829346,
+      "step": 1005
+    },
+    {
+      "epoch": 0.8367854183927091,
+      "grad_norm": 0.7665302753448486,
+      "learning_rate": 7.980029476168944e-06,
+      "loss": 0.432415246963501,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8409279204639603,
+      "grad_norm": 0.6077792048454285,
+      "learning_rate": 7.59248612663479e-06,
+      "loss": 0.3501024007797241,
+      "step": 1015
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 0.882154643535614,
+      "learning_rate": 7.213814650683887e-06,
+      "loss": 0.47015953063964844,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8492129246064622,
+      "grad_norm": 0.423498272895813,
+      "learning_rate": 6.844094268368484e-06,
+      "loss": 0.387217116355896,
+      "step": 1025
+    },
+    {
+      "epoch": 0.8533554266777134,
+      "grad_norm": 0.5810433030128479,
+      "learning_rate": 6.483402327125232e-06,
+      "loss": 0.44982328414916994,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8574979287489644,
+      "grad_norm": 0.39362671971321106,
+      "learning_rate": 6.131814285593707e-06,
+      "loss": 0.3725650548934937,
+      "step": 1035
+    },
+    {
+      "epoch": 0.8616404308202155,
+      "grad_norm": 0.41626113653182983,
+      "learning_rate": 5.7894036978301035e-06,
+      "loss": 0.44669065475463865,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8657829328914665,
+      "grad_norm": 1.006791353225708,
+      "learning_rate": 5.4562421979192925e-06,
+      "loss": 0.4457366466522217,
+      "step": 1045
+    },
+    {
+      "epoch": 0.8699254349627175,
+      "grad_norm": 0.502176821231842,
+      "learning_rate": 5.132399484988615e-06,
+      "loss": 0.42588119506835936,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8740679370339686,
+      "grad_norm": 0.4030306935310364,
+      "learning_rate": 4.817943308626488e-06,
+      "loss": 0.4012446403503418,
+      "step": 1055
+    },
+    {
+      "epoch": 0.8782104391052196,
+      "grad_norm": 0.573884904384613,
+      "learning_rate": 4.512939454708776e-06,
+      "loss": 0.44045190811157225,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 0.4829924404621124,
+      "learning_rate": 4.2174517316361075e-06,
+      "loss": 0.4151927947998047,
+      "step": 1065
+    },
+    {
+      "epoch": 0.8864954432477217,
+      "grad_norm": 0.4790762960910797,
+      "learning_rate": 3.93154195698478e-06,
+      "loss": 0.4421385288238525,
+      "step": 1070
+    },
+    {
+      "epoch": 0.8906379453189727,
+      "grad_norm": 0.36565345525741577,
+      "learning_rate": 3.655269944574202e-06,
+      "loss": 0.41750688552856446,
+      "step": 1075
+    },
+    {
+      "epoch": 0.8947804473902237,
+      "grad_norm": 0.724289059638977,
+      "learning_rate": 3.388693491953576e-06,
+      "loss": 0.4673739433288574,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8989229494614748,
+      "grad_norm": 0.9092391729354858,
+      "learning_rate": 3.1318683683102754e-06,
+      "loss": 0.3644559860229492,
+      "step": 1085
+    },
+    {
+      "epoch": 0.9030654515327258,
+      "grad_norm": 0.679870069026947,
+      "learning_rate": 2.8848483028026895e-06,
+      "loss": 0.4468817710876465,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9072079536039768,
+      "grad_norm": 1.737008810043335,
+      "learning_rate": 2.647684973319814e-06,
+      "loss": 0.35079207420349123,
+      "step": 1095
+    },
+    {
+      "epoch": 0.9113504556752279,
+      "grad_norm": 0.597590446472168,
+      "learning_rate": 2.4204279956698995e-06,
+      "loss": 0.40719895362854003,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9154929577464789,
+      "grad_norm": 0.5171096920967102,
+      "learning_rate": 2.2031249132006547e-06,
+      "loss": 0.42395763397216796,
+      "step": 1105
+    },
+    {
+      "epoch": 0.9196354598177299,
+      "grad_norm": 0.9238699078559875,
+      "learning_rate": 1.9958211868528474e-06,
+      "loss": 0.38481290340423585,
+      "step": 1110
+    },
+    {
+      "epoch": 0.923777961888981,
+      "grad_norm": 0.5502796173095703,
+      "learning_rate": 1.7985601856496947e-06,
+      "loss": 0.40758719444274905,
+      "step": 1115
+    },
+    {
+      "epoch": 0.927920463960232,
+      "grad_norm": 0.5856905579566956,
+      "learning_rate": 1.6113831776238097e-06,
+      "loss": 0.41455903053283694,
+      "step": 1120
+    },
+    {
+      "epoch": 0.932062966031483,
+      "grad_norm": 0.4448857605457306,
+      "learning_rate": 1.4343293211837195e-06,
+      "loss": 0.3878758907318115,
+      "step": 1125
+    },
+    {
+      "epoch": 0.9362054681027341,
+      "grad_norm": 0.42990583181381226,
+      "learning_rate": 1.2674356569217282e-06,
+      "loss": 0.3641160011291504,
+      "step": 1130
+    },
+    {
+      "epoch": 0.9403479701739851,
+      "grad_norm": 0.6104210019111633,
+      "learning_rate": 1.1107370998648058e-06,
+      "loss": 0.4024786472320557,
+      "step": 1135
+    },
+    {
+      "epoch": 0.9444904722452361,
+      "grad_norm": 0.8527662754058838,
+      "learning_rate": 9.64266432170219e-07,
+      "loss": 0.452549409866333,
+      "step": 1140
+    },
+    {
+      "epoch": 0.9486329743164872,
+      "grad_norm": 0.6835286021232605,
+      "learning_rate": 8.280542962673166e-07,
+      "loss": 0.42240447998046876,
+      "step": 1145
+    },
+    {
+      "epoch": 0.9527754763877382,
+      "grad_norm": 0.6754205226898193,
+      "learning_rate": 7.021291884469649e-07,
+      "loss": 0.38313636779785154,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9569179784589892,
+      "grad_norm": 1.0334895849227905,
+      "learning_rate": 5.865174528999706e-07,
+      "loss": 0.42669315338134767,
+      "step": 1155
+    },
+    {
+      "epoch": 0.9610604805302403,
+      "grad_norm": 0.5506278872489929,
+      "learning_rate": 4.812432762057673e-07,
+      "loss": 0.36047968864440916,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9652029826014913,
+      "grad_norm": 0.5626630187034607,
+      "learning_rate": 3.8632868227242435e-07,
+      "loss": 0.41873970031738283,
+      "step": 1165
+    },
+    {
+      "epoch": 0.9693454846727423,
+      "grad_norm": 0.5406614542007446,
+      "learning_rate": 3.017935277291495e-07,
+      "loss": 0.3593956708908081,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9734879867439934,
+      "grad_norm": 0.605750560760498,
+      "learning_rate": 2.2765549777217788e-07,
+      "loss": 0.3540114641189575,
+      "step": 1175
+    },
+    {
+      "epoch": 0.9776304888152444,
+      "grad_norm": 0.5964953899383545,
+      "learning_rate": 1.639301024649209e-07,
+      "loss": 0.4429689884185791,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9817729908864954,
+      "grad_norm": 0.6756887435913086,
+      "learning_rate": 1.1063067349318946e-07,
+      "loss": 0.36358070373535156,
+      "step": 1185
+    },
+    {
+      "epoch": 0.9859154929577465,
+      "grad_norm": 0.49958691000938416,
+      "learning_rate": 6.776836137615262e-08,
+      "loss": 0.3598761081695557,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9900579950289975,
+      "grad_norm": 0.42558741569519043,
+      "learning_rate": 3.535213313355934e-08,
+      "loss": 0.39934353828430175,
+      "step": 1195
+    },
+    {
+      "epoch": 0.9942004971002486,
+      "grad_norm": 1.0632734298706055,
+      "learning_rate": 1.3388770409805685e-08,
+      "loss": 0.440138053894043,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9983429991714996,
+      "grad_norm": 1.1033124923706055,
+      "learning_rate": 1.8828680551918888e-09,
+      "loss": 0.3676408052444458,
+      "step": 1205
+    },
+    {
+      "epoch": 1.0,
+      "step": 1207,
+      "total_flos": 2.5323923576872166e+17,
+      "train_loss": 0.4422070802295119,
+      "train_runtime": 20343.89,
+      "train_samples_per_second": 0.059,
+      "train_steps_per_second": 0.059
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1207,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5323923576872166e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/training_args.bin b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..0fcfde7e443ae48b8b557ee91f5f81ce5c012e6f
--- /dev/null
+++ b/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69e8d3b6cf0314d00b4f9e10bb604fe43b130ceef8e9537712881eeb3377d4eb
+size 5265
diff --git a/MOSS_Speaker-RoPE/problem.txt b/MOSS_Speaker-RoPE/problem.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4b92b6f1da64ec6a88028e316f5a5272cfea4027
--- /dev/null
+++ b/MOSS_Speaker-RoPE/problem.txt
@@ -0,0 +1,8 @@
+- 训练：单次全序列 forward，speaker_input_values 存在 → build_speaker_rotary → attention 走交插分支（is_decode=False），audio token 奇数平面转 θ=W_l·e。
+- 推理 prefill：prepare_inputs_for_generation 第一步会传 speaker_input_values（613-618），条件同训练完全相同 → 走同一条分支，用的也是 checkpoint 里学到的同一组 W_l 和 speaker encoder 权重。audio token 的 K 带着 speaker 角写入 KV cache 后不再变。✓
+但 decode 生成的文本 token：不一致，且会污染对 audio K 的读取。
+ 	训练时（target 文本 token）
+偶数平面	位置 RoPE
+奇数平面	恒等（θ=0）
+后果：decode query 与 audio key 在奇数平面的相对角，训练时是 0 − θ_spk = −θ_spk，推理时变成 m·f_odd − θ_spk。多出的 m·f_odd 随生成位置 m 增长——即模型读 speaker 信息的方式和训练时不一样，且越往后偏得越多。
+所以严格说：speaker 旋转的"写入侧"（audio K）训练推理一致，"读出侧"（生成 token 的 Q）不一致。要一致，decode 路径也应按交插只转偶数平面、奇数平面保持恒等（93 行不能直接用整头 cos/sin）。
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/prompt.txt b/MOSS_Speaker-RoPE/prompt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3447a857c0734bb16e96ac67cd4dc7129ab3d985
--- /dev/null
+++ b/MOSS_Speaker-RoPE/prompt.txt
@@ -0,0 +1,21 @@
+CUDA_VISIBLE_DEVICES=7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+python finetune.py \
+  --train_jsonl data/train_40min.jsonl \
+  --output_dir output \
+  --per_device_train_batch_size 1 \
+  --num_train_epochs 1 \
+  --learning_rate 1e-4 \
+  --warmup_ratio 0.1 \
+  --lr_scheduler_type cosine \
+  --bf16 \
+  --gradient_checkpointing \
+  --attn_implementation flash_attention_2 \
+  --max_length 81920 \
+  --model_name_or_path /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize \
+  --speaker_rotary_std 0.02 \
+  --logging_steps 5 \
+  --logging_strategy steps \
+  --save_strategy steps \
+  --save_steps 200 \
+  --save_total_limit 2 \
+  --report_to none
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/run_train.sh b/MOSS_Speaker-RoPE/run_train.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5e3dc647c7ce9969b99db8e08df6eddfa71e6704
--- /dev/null
+++ b/MOSS_Speaker-RoPE/run_train.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=0
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+cd /wangshuai/moss/MOSS_Speaker-RoPE
+/wangshuai/tools/miniconda3/envs/moss-transcribe-diarize/bin/python finetune.py \
+  --train_jsonl /wangshuai/moss/MOSS_Speaker-RoPE/data/train_40min_new.jsonl \
+  --output_dir /wangshuai/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back \
+  --per_device_train_batch_size 1 \
+  --num_train_epochs 1 \
+  --learning_rate 1e-5 \
+  --warmup_ratio 0.1 \
+  --lr_scheduler_type cosine \
+  --bf16 \
+  --gradient_checkpointing \
+  --max_length 81920 \
+  --model_name_or_path /wangshuai/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5 \
+  --logging_steps 5 \
+  --logging_strategy steps \
+  --save_strategy steps \
+  --save_steps 400 \
+  --save_total_limit 1 \
+  --report_to none \
+  > /wangshuai/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5_back/train.log 2>&1 &
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/run_train_lr1e-4.sh b/MOSS_Speaker-RoPE/run_train_lr1e-4.sh
new file mode 100644
index 0000000000000000000000000000000000000000..4ffa90a3d309e50d1cf5534825b626c4a0de2c79
--- /dev/null
+++ b/MOSS_Speaker-RoPE/run_train_lr1e-4.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=6
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+cd /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize
+/taoye/miniconda3/envs/moss-transcribe-diarize/bin/python finetune.py \
+  --train_jsonl /taoye/lhy/czy/moss/MOSS_Speaker-RoPE/data/train_40min.jsonl \
+  --output_dir /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/output_lr1e-4 \
+  --per_device_train_batch_size 1 \
+  --num_train_epochs 1 \
+  --learning_rate 1e-4 \
+  --warmup_ratio 0.1 \
+  --lr_scheduler_type cosine \
+  --bf16 \
+  --gradient_checkpointing \
+  --attn_implementation flash_attention_2 \
+  --max_length 81920 \
+  --model_name_or_path /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize \
+  --logging_steps 5 \
+  --logging_strategy steps \
+  --save_strategy no \
+  --report_to none
diff --git a/MOSS_Speaker-RoPE/run_train_spk_odd8_s3.sh b/MOSS_Speaker-RoPE/run_train_spk_odd8_s3.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c8443aa4500dd5c4d3f231da78eb0466178753be
--- /dev/null
+++ b/MOSS_Speaker-RoPE/run_train_spk_odd8_s3.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Speaker rotary planes: {3, 11, 19, 27, 35, 43, 51, 59} (8 odd planes, d_s=16)
+# 56 position planes + 8 speaker planes among 64 total (head_dim=128)
+
+source "$(conda info --base)/etc/profile.d/conda.sh"
+conda activate moss-transcribe-diarize
+
+CUDA_VISIBLE_DEVICES=7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+python finetune.py \
+  --train_jsonl data/train_40min.jsonl \
+  --output_dir output_spk_odd8_s3 \
+  --per_device_train_batch_size 1 \
+  --num_train_epochs 1 \
+  --learning_rate 1e-4 \
+  --warmup_ratio 0.1 \
+  --lr_scheduler_type cosine \
+  --bf16 \
+  --gradient_checkpointing \
+  --attn_implementation flash_attention_2 \
+  --max_length 81920 \
+  --model_name_or_path /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize \
+  --speaker_rotary_std 0.02 \
+  --speaker_plane_indices 3,11,19,27,35,43,51,59 \
+  --logging_steps 5 \
+  --logging_strategy steps \
+  --save_strategy steps \
+  --save_steps 200 \
+  --save_total_limit 2 \
+  --report_to none
diff --git a/MOSS_Speaker-RoPE/run_train_spk_odd8_s3_postnorm.sh b/MOSS_Speaker-RoPE/run_train_spk_odd8_s3_postnorm.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3d7b73ba895832e4634a7dd9c59527ae1bbdfdf2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/run_train_spk_odd8_s3_postnorm.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Post-norm training from scratch on GPU7.
+# Normalization: raw encoder output → project → normalize(theta).
+
+source "$(conda info --base)/etc/profile.d/conda.sh"
+conda activate moss-transcribe-diarize
+
+CUDA_VISIBLE_DEVICES=7 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+python finetune.py \
+  --train_jsonl data/train_40min.jsonl \
+  --output_dir output_spk_odd8_s3_postnorm \
+  --per_device_train_batch_size 1 \
+  --num_train_epochs 1 \
+  --learning_rate 1e-4 \
+  --warmup_ratio 0.1 \
+  --lr_scheduler_type cosine \
+  --bf16 \
+  --gradient_checkpointing \
+  --attn_implementation flash_attention_2 \
+  --max_length 81920 \
+  --model_name_or_path /taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize \
+  --speaker_rotary_std 0.02 \
+  --speaker_plane_indices 3,11,19,27,35,43,51,59 \
+  --logging_steps 5 \
+  --logging_strategy steps \
+  --save_strategy steps \
+  --save_steps 200 \
+  --save_total_limit 2 \
+  --report_to none
+EOF
+chmod +x /taoye/lhy/czy/moss/MOSS_Speaker-RoPE/run_train_spk_odd8_s3_postnorm.sh
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/task b/MOSS_Speaker-RoPE/task
new file mode 100644
index 0000000000000000000000000000000000000000..aea87420b917cb4f0344a02820521c0d5cbec5ac
--- /dev/null
+++ b/MOSS_Speaker-RoPE/task
@@ -0,0 +1,11 @@
+基于/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize 这个repo
+即/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/run_train.sh的训练
+还有/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/infer.py的推理
+你可以了解MOSS-Transcribe-Diarize的训练推理网络与配置
+我现在需要将RoPE改为Speaker Encoder + RoPE
+具体修改思路参照/taoye/lhy/czy/moss/speaker_rope.md
+但是有一个问题是/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/wespeaker-voxceleb-redimnet2-B6-LM
+这个模型是wespeaker的模型，用于输出段级别spk embedding
+所以在做speaker 编码的时候需要做一下变通，即每25帧（2s） speaker encoding是一样的，使用2s提取的embedding来作为e，去编码25帧audio feature，其余和MOSS-Transcribe-Diarize一样保持不变
+
+请基于我的要求，帮我完成/taoye/lhy/czy/moss/MOSS_Speaker-RoPE 这个repo
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/tests/diag_speaker.py b/MOSS_Speaker-RoPE/tests/diag_speaker.py
new file mode 100644
index 0000000000000000000000000000000000000000..82e69fd8a309e34e7528bce7c802b0a7ca7f4829
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/diag_speaker.py
@@ -0,0 +1,60 @@
+"""Diagnose speaker encoder output: check that same-speaker windows cluster and
+embedding quality is reasonable. Also show theta distribution for the first sample."""
+
+import os, sys, json
+os.environ.setdefault("CUDA_VISIBLE_DEVICES", "6")
+sys.path.insert(0, "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE")
+
+import numpy as np
+import soundfile as sf
+import soxr
+import torch
+from pathlib import Path
+from finetune import load_model, MossSpeakerRopeProcessor, ScriptArguments
+
+args = ScriptArguments()
+args.attn_implementation = "flash_attention_2"
+model = load_model(args, torch.bfloat16).to("cuda").eval()
+
+line = json.loads(open("/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/data/train_40min.jsonl").readline())
+path = line["conversation"][1]["content"]
+audio, sr = sf.read(path, dtype="float32", always_2d=True)
+audio = audio.mean(axis=1)
+if sr != 16000:
+    audio = soxr.resample(audio, sr, 16000)
+
+winsize = 32000
+n_win = (len(audio) + winsize - 1) // winsize
+padded = np.zeros(n_win * winsize, np.float32)
+padded[:len(audio)] = audio
+wavs = torch.from_numpy(padded.reshape(n_win, winsize)).cuda()
+
+with torch.no_grad():
+    embs = model.model.speaker_encoder(wavs).cpu().numpy()
+
+# cosine matrix for first 12 windows + a few far apart
+n = min(200, n_win)
+embs_n = embs[:n]
+sim = embs_n @ embs_n.T
+diag = np.diag(sim)
+
+# avg similarity between adjacent windows vs distant windows
+adj = np.diag(sim, 1).mean()
+far = (sim[:n-3, 3:]).mean()
+print(f"n_windows={n_win}  embed_dim={embs.shape[1]}")
+print(f"adjacent-cosine(mean)={adj:+.4f}  offset-3-cosine(mean)={far:+.4f}")
+print(f"max-off-diag-sim={sim[np.triu_indices(n,1)].max():.4f}")
+print(f"diag norm={diag.mean():.4f}+-{diag.std():.4f}")
+print(f"std(e)={embs.std(axis=0).mean():.4f}")
+
+# theta magnitudes for this sample
+with torch.no_grad():
+    processor = MossSpeakerRopeProcessor.from_pretrained(args.model_name_or_path)
+    prompt = processor.apply_chat_template([{"role":"user","content":[{"type":"audio","audio":path},{"type":"text","text":"test"}]}], tokenize=False, add_generation_prompt=True)
+    batch = processor(text=prompt+ " test"+processor.tokenizer.eos_token, audio=[audio[:16*16000]], max_length=81920, return_tensors="pt")
+    batch = {k:v.to("cuda") for k,v in batch.items()}
+    spk_cos, spk_sin = model.model.build_speaker_rotary(**{k: batch[k] for k in ["input_ids","inputs_embeds","speaker_input_values","speaker_chunk_mapping","audio_feature_lengths","audio_chunk_mapping"]})
+
+print(f"\nspeaker cos shape: {spk_cos.shape}")
+print(f"spk_cos: min={spk_cos.min():.3f} max={spk_cos.max():.3f} mean={spk_cos.mean():.3f} std={spk_cos.std():.3f}")
+print(f"theta range from acos: [{torch.acos(spk_cos.clamp(-1,1)).min():.1f}, {torch.acos(spk_cos.clamp(-1,1)).max():.1f}]")
diff --git a/MOSS_Speaker-RoPE/tests/infer_both.py b/MOSS_Speaker-RoPE/tests/infer_both.py
new file mode 100644
index 0000000000000000000000000000000000000000..c301c2c5d74963e9adfac42636b8e92dbabd585d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/infer_both.py
@@ -0,0 +1,52 @@
+import json, os, sys
+sys.path.insert(0, '/taoye/lhy/czy/moss/MOSS_Speaker-RoPE')
+os.environ['CUDA_VISIBLE_DEVICES'] = '6'
+import torch, soundfile as sf
+from pathlib import Path
+from moss_speaker_rope.configuration_moss_speaker_rope import MossSpeakerRopeConfig
+from moss_speaker_rope.modeling_moss_speaker_rope import MossSpeakerRopeForConditionalGeneration
+from moss_speaker_rope.processing_moss_speaker_rope import MossSpeakerRopeProcessor
+from moss_speaker_rope.inference_utils import build_transcription_messages
+
+device = torch.device('cuda:0')
+dtype = torch.bfloat16
+BASE = '/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize'
+AUDIO = '/taoye/lhy/datasets/CALLHOME/en/test/wav/callhome_en_test_0013.wav'
+
+processor = MossSpeakerRopeProcessor.from_pretrained(BASE, trust_remote_code=True)
+sfr = int(processor.feature_extractor.sampling_rate)
+audio, sr = sf.read(AUDIO, dtype='float32', always_2d=True); audio = audio.mean(axis=1)
+if sr != sfr: import soxr; audio = soxr.resample(audio, sr, sfr)
+msgs = build_transcription_messages(AUDIO)
+text = processor.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
+
+for lr_tag, CKPT in [('lr1e-4', '/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_lr1e4'),
+                       ('lr2e-4', '/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_lr2e4')]:
+    print(f'=== {lr_tag} ===', flush=True)
+    config_json = json.loads((Path(CKPT) / 'config.json').read_text())
+    for k in ('architectures','auto_map','model_type','dtype','transformers_version'): config_json.pop(k, None)
+    config = MossSpeakerRopeConfig(**config_json)
+    model = MossSpeakerRopeForConditionalGeneration.from_pretrained(
+        CKPT, config=config, dtype=dtype, trust_remote_code=True).to(device).eval()
+    model.model.speaker_encoder.float()
+    batch = processor(text=text, audio=[audio], max_length=81920, return_tensors='pt')
+    batch = {k: v.to(device) for k, v in batch.items()}
+    prompt_len = batch['attention_mask'].sum().item()
+    print(f'prompt_len={prompt_len}, generating...', flush=True)
+    with torch.inference_mode(), torch.amp.autocast('cuda', dtype=dtype):
+        out = model.generate(
+            input_ids=batch['input_ids'], attention_mask=batch['attention_mask'],
+            input_features=batch['input_features'], audio_feature_lengths=batch['audio_feature_lengths'],
+            audio_chunk_mapping=batch['audio_chunk_mapping'],
+            speaker_input_values=batch['speaker_input_values'],
+            speaker_chunk_mapping=batch['speaker_chunk_mapping'],
+            max_new_tokens=100, do_sample=False,
+        )
+    gen_text = processor.tokenizer.decode(out[0][prompt_len:], skip_special_tokens=True)
+    path = Path('/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/infer_results') / f'transcript_{lr_tag}.txt'
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(gen_text, encoding='utf-8')
+    print(f'{lr_tag}: {len(gen_text)} chars, preview: {repr(gen_text[:200])}', flush=True)
+    del model; torch.cuda.empty_cache()
+
+print('DONE', flush=True)
diff --git a/MOSS_Speaker-RoPE/tests/infer_compare.py b/MOSS_Speaker-RoPE/tests/infer_compare.py
new file mode 100644
index 0000000000000000000000000000000000000000..965b41a3da13575beb9015e43ad95f2fc4034f19
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/infer_compare.py
@@ -0,0 +1,56 @@
+"""Compare lr=1e-4 vs lr=2e-4 on AISHELL-4 test audio."""
+
+import json, os, sys
+sys.path.insert(0, "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE")
+os.environ["CUDA_VISIBLE_DEVICES"] = "6"
+os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+os.environ["TRUST_REMOTE_CODE"] = "1"
+
+from pathlib import Path
+import torch
+
+from moss_speaker_rope import parse_transcript
+from moss_speaker_rope.configuration_moss_speaker_rope import MossSpeakerRopeConfig
+from moss_speaker_rope.inference_utils import (
+    build_transcription_messages, generate_transcription, resolve_device,
+)
+from moss_speaker_rope.modeling_moss_speaker_rope import MossSpeakerRopeForConditionalGeneration
+from moss_speaker_rope.processing_moss_speaker_rope import MossSpeakerRopeProcessor
+
+BASE = "/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize"
+AUDIO = "/taoye/lhy/datasets/CALLHOME/en/test/wav/callhome_en_test_0013.wav"
+OUT = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/infer_results"
+
+device = resolve_device("auto")
+dtype = torch.bfloat16
+processor = MossSpeakerRopeProcessor.from_pretrained(BASE, trust_remote_code=True)
+messages = build_transcription_messages(AUDIO)
+
+for lr_tag, ckpt_dir in [("lr1e-4", "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_lr1e4"),
+                          ("lr2e-4", "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_lr2e4")]:
+    print(f"\n=== {lr_tag} ===")
+    config_json = json.loads((Path(ckpt_dir) / "config.json").read_text())
+    for k in ("architectures", "auto_map", "model_type", "dtype", "transformers_version"):
+        config_json.pop(k, None)
+    config = MossSpeakerRopeConfig(**config_json)
+    model = MossSpeakerRopeForConditionalGeneration.from_pretrained(
+        ckpt_dir, config=config, dtype=dtype, trust_remote_code=True
+    ).to(device).eval()
+    model.model.speaker_encoder.float()
+    result = generate_transcription(
+        model, processor, messages, max_new_tokens=4096, do_sample=False, device=device, dtype=dtype
+    )
+    path = Path(OUT) / f"transcript_{lr_tag}.txt"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        f.write(result["text"])
+    print(f"Saved to {path} ({len(result['text'])} chars)")
+    del model; torch.cuda.empty_cache()
+
+print("\n=== SEGMENTS lr=1e-4 ===")
+for seg in parse_transcript((Path(OUT) / "transcript_lr1e-4.txt").read_text()):
+    print(f"[{seg.start:.1f}-{seg.end:.1f}] {seg.speaker}: {seg.text}")
+
+print("\n=== SEGMENTS lr=2e-4 ===")
+for seg in parse_transcript((Path(OUT) / "transcript_lr2e-4.txt").read_text()):
+    print(f"[{seg.start:.1f}-{seg.end:.1f}] {seg.speaker}: {seg.text}")
diff --git a/MOSS_Speaker-RoPE/tests/nan_backward.py b/MOSS_Speaker-RoPE/tests/nan_backward.py
new file mode 100644
index 0000000000000000000000000000000000000000..babb2639e4d37584b2654dd0f54f36f783b408ce
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/nan_backward.py
@@ -0,0 +1,64 @@
+"""Check backward gradients batch by batch in the real training order."""
+
+import os, sys
+os.environ.setdefault("CUDA_VISIBLE_DEVICES", "6")
+sys.path.insert(0, "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE")
+
+import torch
+from finetune import ConversationDataset, DataCollator, ScriptArguments, load_model
+from moss_speaker_rope.processing_moss_speaker_rope import MossSpeakerRopeProcessor
+from transformers import TrainingArguments, Trainer, set_seed
+
+args = ScriptArguments(
+    train_jsonl="/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/data/train_40min.jsonl",
+    max_length=81920,
+    attn_implementation="flash_attention_2",
+)
+targs = TrainingArguments(
+    output_dir="/tmp/opencode/nan_repro", per_device_train_batch_size=1, bf16=True,
+    gradient_checkpointing=True, report_to=[], save_strategy="no", seed=42,
+    remove_unused_columns=False,
+)
+
+set_seed(42)
+processor = MossSpeakerRopeProcessor.from_pretrained(args.model_name_or_path)
+dataset = ConversationDataset(args.train_jsonl)
+collator = DataCollator(processor, args.max_length)
+
+from transformers import modeling_utils
+modeling_utils.PreTrainedModel._flash_attn_can_dispatch = lambda *a, **kw: True
+model = load_model(args, torch.bfloat16)
+model.config.use_cache = False
+model.config.text_config.use_cache = False
+model = model.to("cuda")
+model.gradient_checkpointing_enable()
+model.model.speaker_encoder.gradient_checkpointing = True
+model.train()
+
+trainer = Trainer(model=model, args=targs, train_dataset=dataset, data_collator=collator, processing_class=processor)
+loader = trainer.get_train_dataloader()
+
+for step, batch in enumerate(loader):
+    if step >= 6:
+        break
+    batch = {k: v.to("cuda") for k, v in batch.items()}
+    loss = model(**batch).loss
+    loss.backward()
+    report = {}
+    for name, module in [
+        ("spk_enc", model.model.speaker_encoder),
+        ("whisper", model.model.whisper_encoder),
+        ("adaptor", model.model.vq_adaptor),
+        ("llm", model.model.language_model),
+    ]:
+        sq, n_nan = 0.0, 0
+        for p in module.parameters():
+            if p.grad is not None:
+                g = p.grad.float()
+                if not torch.isfinite(g).all():
+                    n_nan += 1
+                else:
+                    sq += g.norm().item() ** 2
+        report[name] = f"{sq**0.5:9.2f}/nan:{n_nan}"
+    print(f"step={step} loss={loss.item():8.4f} " + " ".join(f"{k}={v}" for k, v in report.items()), flush=True)
+    model.zero_grad(set_to_none=True)
diff --git a/MOSS_Speaker-RoPE/tests/nan_repro.py b/MOSS_Speaker-RoPE/tests/nan_repro.py
new file mode 100644
index 0000000000000000000000000000000000000000..0046b0955db884f963bc413f83590d78cdf27e72
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/nan_repro.py
@@ -0,0 +1,60 @@
+"""Reproduce NaN: iterate the exact shuffled training order and find the bad batch."""
+
+import os, sys
+os.environ.setdefault("CUDA_VISIBLE_DEVICES", "6")
+sys.path.insert(0, "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE")
+
+import json
+import torch
+from pathlib import Path
+
+from finetune import ConversationDataset, DataCollator, ScriptArguments, load_model
+from moss_speaker_rope.processing_moss_speaker_rope import MossSpeakerRopeProcessor
+from transformers import TrainingArguments, Trainer, set_seed
+
+args = ScriptArguments(
+    train_jsonl="/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/data/train_40min.jsonl",
+    max_length=81920,
+    attn_implementation="flash_attention_2",
+)
+targs = TrainingArguments(
+    output_dir="/tmp/opencode/nan_repro", per_device_train_batch_size=1, bf16=True,
+    gradient_checkpointing=True, report_to=[], save_strategy="no", seed=42,
+    remove_unused_columns=False,
+)
+
+set_seed(42)
+processor = MossSpeakerRopeProcessor.from_pretrained(args.model_name_or_path)
+dataset = ConversationDataset(args.train_jsonl)
+collator = DataCollator(processor, args.max_length)
+
+from transformers import modeling_utils
+modeling_utils.PreTrainedModel._flash_attn_can_dispatch = lambda *a, **kw: True
+model = load_model(args, torch.bfloat16)
+model.config.use_cache = False
+model.config.text_config.use_cache = False
+model = model.to("cuda").train()
+
+trainer = Trainer(model=model, args=targs, train_dataset=dataset, data_collator=collator, processing_class=processor)
+loader = trainer.get_train_dataloader()
+
+samples_order = list(torch.utils.data.RandomSampler(dataset, generator=torch.Generator().manual_seed(42)))
+
+with torch.no_grad():
+    for step, batch in enumerate(loader):
+        if step >= 12:
+            break
+        idx = samples_order[step]
+        path = dataset.samples[idx]["audio"]
+        batch = {k: v.to("cuda") for k, v in batch.items()}
+        # check raw inputs
+        bad_wav = not torch.isfinite(batch["speaker_input_values"]).all()
+        bad_feat = not torch.isfinite(batch["input_features"]).all()
+        spk_emb = model.model.speaker_encoder(batch["speaker_input_values"])
+        bad_emb = not torch.isfinite(spk_emb).all()
+        out = model(**batch)
+        loss = out.loss.item()
+        n_lab = int((batch["labels"] != -100).sum())
+        print(f"step={step:2d} idx={idx:4d} loss={loss:9.4f} labels={n_lab:6d} "
+              f"bad_wav={bad_wav} bad_feat={bad_feat} bad_emb={bad_emb} "
+              f"seq={batch['input_ids'].shape[1]} {Path(path).name}", flush=True)
diff --git a/MOSS_Speaker-RoPE/tests/smoke_test.py b/MOSS_Speaker-RoPE/tests/smoke_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..9598a43d06b4b8c6d41c7354b0026cbda418d485
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/smoke_test.py
@@ -0,0 +1,106 @@
+"""Smoke test: forward/loss/backward/generate on a short audio slice (GPU)."""
+
+import os
+import sys
+
+os.environ.setdefault("CUDA_VISIBLE_DEVICES", "6")
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import json
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+
+from moss_speaker_rope.configuration_moss_speaker_rope import MossSpeakerRopeConfig
+from moss_speaker_rope.modeling_moss_speaker_rope import MossSpeakerRopeForConditionalGeneration
+from moss_speaker_rope.processing_moss_speaker_rope import MossSpeakerRopeProcessor
+from moss_speaker_rope.inference_utils import build_transcription_messages
+
+BASE = "/taoye/lhy/czy/moss/MOSS-Transcribe-Diarize/MOSS-Transcribe-Diarize"
+SPK = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/wespeaker-voxceleb-redimnet2-B6-LM"
+WAV = "/taoye/lhy/datasets/AISHELL-ASR0055/AISHELL-ASR0055/M/M_R001/wav/20200616_M_R001S01C01.wav"
+
+device = torch.device("cuda:0")
+
+print("== load processor ==")
+processor = MossSpeakerRopeProcessor.from_pretrained(BASE)
+print("speaker_window_samples:", processor.speaker_window_samples)
+
+print("== load model ==")
+base_config = json.loads((Path(BASE) / "config.json").read_text())
+for key in ("architectures", "auto_map", "model_type", "dtype", "transformers_version"):
+    base_config.pop(key, None)
+config = MossSpeakerRopeConfig(**base_config)
+model = MossSpeakerRopeForConditionalGeneration.from_pretrained(BASE, config=config, dtype=torch.bfloat16)
+model.model.speaker_encoder.float()
+model.model.speaker_encoder.load_wespeaker_checkpoint(SPK)
+model = model.to(device)
+print("partial_rotary_factor:", config.text_config.partial_rotary_factor)
+print("pos rotary inv_freq:", model.model.language_model.rotary_emb.inv_freq.shape)
+
+print("== build inputs (33s slice) ==")
+audio, sr = sf.read(WAV, dtype="float32", always_2d=True)
+audio = audio.mean(axis=1)[: 33 * sr]
+prompt = processor.apply_chat_template(
+    build_transcription_messages(WAV), tokenize=False, add_generation_prompt=True
+)
+target = "[0.19][S01]零零二[1.28][1.31][S02]零零五[2.21]"
+text = prompt + target + processor.tokenizer.eos_token
+batch = processor(text=text, audio=[audio], max_length=81920, return_tensors="pt")
+for k, v in batch.items():
+    print(k, tuple(v.shape), v.dtype)
+
+n_audio_tokens = int((batch["input_ids"] == config.audio_token_id).sum())
+n_windows = batch["speaker_input_values"].shape[0]
+print(f"audio tokens={n_audio_tokens}, speaker windows={n_windows} (windows*25={n_windows*25})")
+assert n_windows * 25 >= n_audio_tokens
+
+labels = batch["input_ids"].clone()
+prompt_ids = processor.expand_audio_token(prompt, int(batch["audio_feature_lengths"].sum()), 81920)
+labels[0, : len(prompt_ids)] = -100
+batch["labels"] = labels
+
+print("== training-mode forward/backward ==")
+model.train()
+batch_gpu = {k: v.to(device) for k, v in batch.items()}
+out = model(**batch_gpu)
+print("loss:", out.loss.item())
+assert torch.isfinite(out.loss)
+out.loss.backward()
+
+grads = {}
+for name, module in [
+    ("speaker_encoder", model.model.speaker_encoder),
+    ("whisper_encoder", model.model.whisper_encoder),
+    ("language_model", model.model.language_model),
+]:
+    total = 0.0
+    for p in module.parameters():
+        if p.grad is not None:
+            total += p.grad.float().norm().item() ** 2
+    grads[name] = total ** 0.5
+    print(f"grad norm {name}: {grads[name]:.6f}")
+assert grads["speaker_encoder"] > 0, "no gradient reached the speaker encoder!"
+model.zero_grad(set_to_none=True)
+
+print("== eval-mode determinism check ==")
+model.eval()
+with torch.no_grad():
+    l1 = model(**batch_gpu).loss.item()
+    l2 = model(**batch_gpu).loss.item()
+print("eval losses:", l1, l2)
+assert abs(l1 - l2) < 1e-6, "inference W seed not deterministic"
+
+print("== generate ==")
+gen_inputs = {k: v.to(device) for k, v in batch.items() if k != "labels"}
+prompt_len = len(prompt_ids)
+gen_inputs["input_ids"] = gen_inputs["input_ids"][:, :prompt_len]
+gen_inputs["attention_mask"] = gen_inputs["attention_mask"][:, :prompt_len]
+with torch.inference_mode():
+    output = model.generate(**gen_inputs, max_new_tokens=40, do_sample=False)
+gen_text = processor.tokenizer.decode(output[0][prompt_len:], skip_special_tokens=True)
+print("generated:", gen_text)
+
+print("ALL OK")
diff --git a/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap.png b/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap.png
new file mode 100644
index 0000000000000000000000000000000000000000..b11b9a976ce6326ba3c7fbdecbf3b1e7867f5901
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f041e732b4b843250e81660f129e9c335fc2e2f0656cef2e666f9021bb196d25
+size 1579591
diff --git a/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap_norm.png b/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap_norm.png
new file mode 100644
index 0000000000000000000000000000000000000000..2e681a7913d92ae59efeb11c323550ba74bd1826
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap_norm.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4786965c3557bac6090845231fdcf31dc8fce5a643c664278d9452c9d3cbbfb8
+size 1178836
diff --git a/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm.png b/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm.png
new file mode 100644
index 0000000000000000000000000000000000000000..d1c429cd8dd8d214cf792a92a7c85c1af76a34c3
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a3fe80d30f447c149b54cc65717586305c54c4c6beb62bcefd15bf18abbd709
+size 2402575
diff --git a/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm_norm.png b/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm_norm.png
new file mode 100644
index 0000000000000000000000000000000000000000..958189c63544289c35f356fd459e5097ae2fa1f7
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_embed_postnorm_norm.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037e8061bac862e4c55e109d012ad563e03596349786dfd20bf9fe8538ce3f0d
+size 1195445
diff --git a/MOSS_Speaker-RoPE/tests/speaker_postnorm_analysis.png b/MOSS_Speaker-RoPE/tests/speaker_postnorm_analysis.png
new file mode 100644
index 0000000000000000000000000000000000000000..182898b8e69744abf61f0d6a0ab51c4ac7fa0263
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_postnorm_analysis.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72e99087d192c3efdeee55bb95aee3ffdcb04511aa8a6873c3827ebb539dcf62
+size 121637
diff --git a/MOSS_Speaker-RoPE/tests/speaker_spkw5_analysis.png b/MOSS_Speaker-RoPE/tests/speaker_spkw5_analysis.png
new file mode 100644
index 0000000000000000000000000000000000000000..1b5317f86ae18249dc30ea9ee63a1e7ef868db7a
Binary files /dev/null and b/MOSS_Speaker-RoPE/tests/speaker_spkw5_analysis.png differ
diff --git a/MOSS_Speaker-RoPE/tests/speaker_spkw5_heatmap.png b/MOSS_Speaker-RoPE/tests/speaker_spkw5_heatmap.png
new file mode 100644
index 0000000000000000000000000000000000000000..f09f41d46e0d6dcc2c8722fbbc09bc74ace186ba
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/speaker_spkw5_heatmap.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1416769a6a401a42ef34a5d2ef4ba4c837d8e42696098e0d41656e272808b03d
+size 1910544
diff --git a/MOSS_Speaker-RoPE/tests/vis_speaker_embed.py b/MOSS_Speaker-RoPE/tests/vis_speaker_embed.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f6ad6b950ed8d679234a3fbe0323b90f61dc41b
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/vis_speaker_embed.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Visualize raw (192d) vs projected (8d/speaker-plane) speaker embeddings for an audio."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+CKPT = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3"
+AUDIO = "/taoye/lhy/datasets/AISHELL-4/test/wav/L_R003S01C02.wav"
+SAMPLE_RATE = 16000
+WINDOW_SECONDS = 2.0
+CHUNK_BATCH_SIZE = 48
+
+
+def load_model(ckpt: str, device: torch.device):
+    from moss_speaker_rope.configuration_moss_speaker_rope import MossSpeakerRopeConfig
+    from moss_speaker_rope.modeling_moss_speaker_rope import MossSpeakerRopeForConditionalGeneration
+
+    cfg_dict = json.loads((Path(ckpt) / "config.json").read_text())
+    for k in ("architectures", "auto_map", "model_type", "dtype", "transformers_version"):
+        cfg_dict.pop(k, None)
+    config = MossSpeakerRopeConfig(**cfg_dict)
+    model = MossSpeakerRopeForConditionalGeneration.from_pretrained(
+        ckpt, config=config, dtype=torch.bfloat16, low_cpu_mem_usage=False,
+    ).to(device).eval()
+    model.model.speaker_encoder.float()
+    return model
+
+
+def main():
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    model = load_model(CKPT, device)
+
+    # Load audio and split into 2s windows
+    audio, sr = sf.read(AUDIO, dtype="float32", always_2d=True)
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+    if sr != SAMPLE_RATE:
+        import soxr
+        audio = soxr.resample(audio, sr, SAMPLE_RATE)
+
+    window_samples = int(WINDOW_SECONDS * SAMPLE_RATE)
+    n_win = (audio.shape[0] + window_samples - 1) // window_samples
+    padded = np.zeros(n_win * window_samples, dtype=np.float32)
+    padded[: audio.shape[0]] = audio
+    windows = padded.reshape(n_win, window_samples)
+    print(f"Audio: {audio.shape[0]} samples, {n_win} windows, each {window_samples} samples ({WINDOW_SECONDS}s)")
+
+    # --- Extract raw speaker embeddings (192d) ---
+    speaker_encoder = model.model.speaker_encoder
+    speaker_encoder.eval()
+
+    wav_t = torch.from_numpy(windows).to(device)
+    with torch.no_grad():
+        raw_embs = _extract_embeddings(speaker_encoder, wav_t, CHUNK_BATCH_SIZE)
+
+    raw_embs = F.normalize(raw_embs.float(), dim=-1).cpu().numpy()
+    sim_raw = raw_embs @ raw_embs.T  # cosine similarity (normalized)
+
+    # --- Extract projected embeddings (θ for each layer) ---
+    projections = model.model.speaker_projections  # list of (8, 192)
+    proj_layers = {}  # layer_idx -> (n_win, 8) projected θ
+    for layer_idx, W in enumerate(projections):
+        W_f32 = W.data.float().cpu().numpy()  # (8, 192)
+        theta = raw_embs @ W_f32.T  # (n_win, 192) @ (192, 8) = (n_win, 8)
+        proj_layers[layer_idx] = theta
+
+    # --- Plot ---
+    n_layers = len(projections)
+    plot_layers = [0, 7, 14, 21, 27]  # show 5 representative layers
+    n_cols = 1 + len(plot_layers)
+    fig, axes = plt.subplots(1, n_cols, figsize=(4 * n_cols, 3.8))
+
+    # Raw embedding similarity
+    im0 = axes[0].imshow(sim_raw, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+    axes[0].set_title(f"Raw Encoder (192d)\n{n_win} windows")
+    axes[0].set_xlabel("Window index")
+    axes[0].set_ylabel("Window index")
+    plt.colorbar(im0, ax=axes[0], fraction=0.046)
+
+    for col, li in enumerate(plot_layers):
+        ax = axes[1 + col]
+        theta = proj_layers[li]  # (n_win, 8)
+        sim_proj = theta @ theta.T  # dot product (not normalized → magnitude matters)
+        # Also show normalized for comparison
+        theta_norm = theta / (np.linalg.norm(theta, axis=1, keepdims=True) + 1e-10)
+        sim_proj_norm = theta_norm @ theta_norm.T
+        im = ax.imshow(sim_proj, cmap="RdBu_r", vmin=None, vmax=None, aspect="auto")
+        ax.set_title("Layer {} projected (8d){}dot product".format(li, chr(10)))
+        ax.set_xlabel("Window index")
+        ax.set_ylabel("Window index")
+        plt.colorbar(im, ax=ax, fraction=0.046)
+
+    fig.suptitle(f"Speaker Embedding Similarity — {Path(AUDIO).stem}", fontsize=13)
+    plt.tight_layout()
+    out_path = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap.png"
+    fig.savefig(out_path, dpi=150, bbox_inches="tight")
+    print(f"Saved: {out_path}")
+
+    # --- Also show normalized projection for one layer ---
+    fig2, axes2 = plt.subplots(1, 2, figsize=(10, 4))
+    sim_raw_show = sim_raw
+    im_a = axes2[0].imshow(sim_raw_show, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+    axes2[0].set_title(f"Raw Encoder similarity (192d, cosine)")
+    plt.colorbar(im_a, ax=axes2[0])
+
+    # Pick middle layer normalized
+    mid_layer = n_layers // 2
+    tn = proj_layers[mid_layer]
+    tn = tn / (np.linalg.norm(tn, axis=1, keepdims=True) + 1e-10)
+    sim_mid = tn @ tn.T
+    im_b = axes2[1].imshow(sim_mid, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+    axes2[1].set_title(f"Layer {mid_layer} projected similarity (8d, cosine)")
+    plt.colorbar(im_b, ax=axes2[1])
+    fig2.suptitle(f"Cosine-normalized — {Path(AUDIO).stem}", fontsize=13)
+    plt.tight_layout()
+    out_path2 = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/tests/speaker_embed_heatmap_norm.png"
+    fig2.savefig(out_path2, dpi=150, bbox_inches="tight")
+    print(f"Saved: {out_path2}")
+
+    plt.close("all")
+
+
+def _extract_embeddings(encoder, wav_chunks, chunk_batch_size):
+    """Extract L2-normalized embeddings from the speaker encoder."""
+    active = wav_chunks.std(dim=-1) > 1e-4
+    all_active = wav_chunks[active]
+    n_active = all_active.shape[0]
+    if n_active == 0:
+        return wav_chunks.new_zeros((wav_chunks.shape[0], encoder.embed_dim))
+
+    outputs = []
+    with torch.autocast(device_type=wav_chunks.device.type, enabled=False):
+        for start in range(0, n_active, chunk_batch_size):
+            batch = all_active[start : start + chunk_batch_size]
+            if batch.shape[0] == 1:
+                batch = batch.repeat(2, 1)
+            emb = encoder._forward_chunk(batch)
+            if batch.shape[0] != all_active[start : start + chunk_batch_size].shape[0]:
+                emb = emb[:1]
+            outputs.append(emb)
+    all_embs = F.normalize(torch.cat(outputs, dim=0).float(), dim=-1)
+    embeddings = wav_chunks.new_zeros((wav_chunks.shape[0], encoder.embed_dim))
+    embeddings[active] = all_embs
+    return embeddings
+
+
+if __name__ == "__main__":
+    main()
diff --git a/MOSS_Speaker-RoPE/tests/vis_speaker_embed_postnorm.py b/MOSS_Speaker-RoPE/tests/vis_speaker_embed_postnorm.py
new file mode 100644
index 0000000000000000000000000000000000000000..55dee1a41e1211be0d7eec408722e02ed6d17765
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/vis_speaker_embed_postnorm.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Visualize raw vs projected speaker embeddings for post-norm checkpoint."""
+
+import json, sys
+from pathlib import Path
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+CKPT = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_v2"
+AUDIO = "/taoye/lhy/datasets/AISHELL-4/test/wav/L_R003S01C02.wav"
+
+from safetensors.torch import load_file
+from moss_speaker_rope.speaker_encoder import SpeakerEncoder
+
+cfg = json.loads((Path(CKPT) / "config.json").read_text())
+spk_args = cfg.get("speaker_encoder_args", {})
+tfmel_args = cfg.get("speaker_tfmel_args", {})
+
+device = torch.device("cuda")
+encoder = SpeakerEncoder(
+    model_args=spk_args, tfmel_args=tfmel_args, sample_rate=cfg.get("speaker_sample_rate", 16000)
+).to(device).float().eval()
+
+state = load_file(str(Path(CKPT) / "model.safetensors"))
+encoder_state = {k[len("model.speaker_encoder."):]: v for k, v in state.items() if k.startswith("model.speaker_encoder.")}
+encoder.load_state_dict(encoder_state, strict=False)
+
+proj_weights = [
+    state[f"model.speaker_projections.{i}"].float().cpu().numpy() for i in range(28)
+]
+
+audio, sr = sf.read(AUDIO, dtype="float32", always_2d=True)
+if audio.ndim > 1:
+    audio = audio.mean(axis=1)
+
+window_samples = 32000
+n_win = (audio.shape[0] + window_samples - 1) // window_samples
+padded = np.zeros(n_win * window_samples, dtype=np.float32)
+padded[: audio.shape[0]] = audio
+windows = padded.reshape(n_win, window_samples)
+
+raw_embs_list = []
+for start in range(0, n_win, 64):
+    batch = torch.from_numpy(windows[start:start+64]).to(device)
+    with torch.no_grad():
+        emb = encoder(batch)
+    raw_embs_list.append(emb.cpu().numpy())
+    del batch
+raw_embs = np.concatenate(raw_embs_list, axis=0)
+print(f"Windows: {n_win}, raw emb shape: {raw_embs.shape}")
+
+sim_raw = raw_embs @ raw_embs.T  # raw dot product (no L2 norm)
+
+# Project through each layer's W, then normalize (matching training)
+proj_norms = {}
+for li in range(28):
+    W = proj_weights[li]  # (8, 192)
+    theta = raw_embs @ W.T  # (N, 8)
+    theta_norm = theta / (np.linalg.norm(theta, axis=1, keepdims=True) + 1e-10)
+    proj_norms[li] = theta_norm
+
+# === Quantitative ===
+print(f"\nRaw encoder (unnormalized, dot product):")
+off = sim_raw[np.triu_indices_from(sim_raw, k=1)]
+print(f"  mean={off.mean():.4f}  std={off.std():.4f}")
+print(f"  #pairs>0: {(off>0).sum():,}  #pairs<0: {(off<0).sum():,}")
+
+print(f"\nProjected + normalized (8d cosine):")
+for li in [0, 7, 14, 21, 27]:
+    sim = proj_norms[li] @ proj_norms[li].T
+    off = sim[np.triu_indices_from(sim, k=1)]
+    print(f"  layer {li:2d}: mean={off.mean():+.4f}  #>0.9={(off>0.9).sum():,}  #<-0.5={(off<-0.5).sum():,}  #<-0.8={(off<-0.8).sum():,}")
+
+# === Figure 1: raw dot-product + projected cosine for 5 layers ===
+plot_layers = [0, 7, 14, 21, 27]
+n_cols = 1 + len(plot_layers)
+fig, axes = plt.subplots(1, n_cols, figsize=(4 * n_cols, 3.8))
+
+im0 = axes[0].imshow(sim_raw, cmap="RdBu_r", aspect="auto")
+axes[0].set_title("Raw Encoder (unnormalized,\ndot product)")
+plt.colorbar(im0, ax=axes[0], fraction=0.046)
+
+for col, li in enumerate(plot_layers):
+    ax = axes[1 + col]
+    sim = proj_norms[li] @ proj_norms[li].T
+    im = ax.imshow(sim, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+    ax.set_title(f"Layer {li} projected\n(8d, L2-normed, cosine)")
+    plt.colorbar(im, ax=ax, fraction=0.046)
+
+fig.suptitle(f"Post-norm Speaker Embeddings — {Path(AUDIO).stem}", fontsize=13)
+plt.tight_layout()
+out_path = Path(__file__).parent / "speaker_embed_postnorm.png"
+fig.savefig(out_path, dpi=150, bbox_inches="tight")
+print(f"Saved: {out_path}")
+
+# === Figure 2: normalized raw vs normalized projected ===
+fig2, axes2 = plt.subplots(1, 2, figsize=(10, 4))
+raw_norm = raw_embs / (np.linalg.norm(raw_embs, axis=1, keepdims=True) + 1e-10)
+im_a = axes2[0].imshow(raw_norm @ raw_norm.T, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+axes2[0].set_title("Raw Encoder (L2-normed, cosine)")
+plt.colorbar(im_a, ax=axes2[0])
+
+mid = proj_norms[14] @ proj_norms[14].T
+im_b = axes2[1].imshow(mid, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+axes2[1].set_title("Layer 14 projected (8d, L2-normed, cosine)")
+plt.colorbar(im_b, ax=axes2[1])
+fig2.suptitle(f"Cosine normalized — {Path(AUDIO).stem}", fontsize=13)
+plt.tight_layout()
+out_path2 = Path(__file__).parent / "speaker_embed_postnorm_norm.png"
+fig2.savefig(out_path2, dpi=150, bbox_inches="tight")
+print(f"Saved: {out_path2}")
+plt.close("all")
+
+# === Compare with pre-norm version ===
+print(f"\n=== Comparison: pre-norm vs post-norm ===")
+print(f"{'metric':<40} {'pre-norm':>12} {'post-norm':>12}")
+# pre-norm data from earlier run (192d L2-normed cosine similarity)
+pre_raw = 0.3294  # mean off-diag cosine sim
+pre_p14 = 0.4399  # layer 14 cosine mean
+
+raw_norm_sim = (raw_norm @ raw_norm.T)[np.triu_indices_from(sim_raw, k=1)].mean()
+p14_sim = proj_norms[14] @ proj_norms[14].T
+p14_off = p14_sim[np.triu_indices_from(p14_sim, k=1)].mean()
+
+print(f"{'Raw encoder cosine sim mean':<40} {pre_raw:>12.4f} {raw_norm_sim:>12.4f}")
+print(f"{'Layer 14 projected cosine mean':<40} {pre_p14:>12.4f} {p14_off:>12.4f}")
diff --git a/MOSS_Speaker-RoPE/tests/vis_spkw5.py b/MOSS_Speaker-RoPE/tests/vis_spkw5.py
new file mode 100644
index 0000000000000000000000000000000000000000..64a0fcae1df46d63e366971793ad6467954af605
--- /dev/null
+++ b/MOSS_Speaker-RoPE/tests/vis_spkw5.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Analyze spkw5 (post-norm + speaker-digit ×5 weighted) checkpoint."""
+import json, sys
+from pathlib import Path
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+CKPT = "/taoye/lhy/czy/moss/MOSS_Speaker-RoPE/output_spk_odd8_s3_postnorm_spkw5/checkpoint-1207"
+AUDIO = "/taoye/lhy/datasets/AISHELL-4/test/wav/L_R003S01C02.wav"
+
+from safetensors.torch import load_file
+from moss_speaker_rope.speaker_encoder import SpeakerEncoder
+
+cfg = json.loads((Path(CKPT) / "config.json").read_text())
+spk_args = cfg.get("speaker_encoder_args", {})
+tfmel_args = cfg.get("speaker_tfmel_args", {})
+
+device = torch.device("cuda")
+encoder = SpeakerEncoder(model_args=spk_args, tfmel_args=tfmel_args, sample_rate=16000).to(device).float().eval()
+state = load_file(str(Path(CKPT) / "model.safetensors"))
+encoder_state = {k[len("model.speaker_encoder."):]: v for k,v in state.items() if k.startswith("model.speaker_encoder.")}
+encoder.load_state_dict(encoder_state, strict=False)
+proj = [state[f"model.speaker_projections.{i}"].float().cpu().numpy() for i in range(28)]
+
+audio, _ = sf.read(AUDIO, dtype="float32", always_2d=True)
+audio = audio.mean(axis=1)
+ws = 32000; n = (audio.shape[0] + ws - 1) // ws
+pad = np.zeros(n * ws, dtype=np.float32); pad[:audio.shape[0]] = audio
+wins = pad.reshape(n, ws)
+
+raw = []
+for s in range(0, n, 64):
+    b = torch.from_numpy(wins[s:s+64]).to(device)
+    with torch.no_grad():
+        raw.append(encoder(b).cpu().numpy())
+    del b
+raw = np.concatenate(raw, axis=0)
+
+raw_norm = raw / (np.linalg.norm(raw, axis=1, keepdims=True) + 1e-10)
+sim_raw = raw_norm @ raw_norm.T
+off_raw = sim_raw[np.triu_indices_from(sim_raw, k=1)]
+near_raw = np.concatenate([np.diag(sim_raw, k=i) for i in range(1, 11)]).mean()
+far_raw  = np.concatenate([np.diag(sim_raw, k=i) for i in range(50, 60)]).mean()
+
+print("=== Raw encoder (192d, L2-normed) ===")
+print(f"  mean={off_raw.mean():.4f}  near={near_raw:.4f}  far={far_raw:.4f}  delta={near_raw-far_raw:+.4f}")
+
+print(f"\n=== All 28 layers: projected (8d, L2-normed) cosine ===")
+print(f"{'layer':>5} {'mean':>8} {'>0.9':>8} {'<-0.8':>8} {'near':>8} {'far':>8} {'delta':>8}")
+print("-"*58)
+layer_stats = []
+for li in range(28):
+    W = proj[li]; theta = raw @ W.T
+    theta = theta / (np.linalg.norm(theta, axis=1, keepdims=True) + 1e-10)
+    sim = theta @ theta.T
+    off = sim[np.triu_indices_from(sim, k=1)]
+    near = np.concatenate([np.diag(sim, k=i) for i in range(1, 11)]).mean()
+    far  = np.concatenate([np.diag(sim, k=i) for i in range(50, 60)]).mean()
+    layer_stats.append({'layer': li, 'mean': off.mean(), 'near': near, 'far': far, 'gt09': (off>0.9).sum(), 'lt_m8': (off<-0.8).sum()})
+    print(f"{li:5d} {off.mean():+8.4f} {(off>0.9).sum():8d} {(off<-0.8).sum():8d} {near:+8.4f} {far:+8.4f} {near-far:+8.4f}")
+
+# Plot
+fig, axes = plt.subplots(2, 2, figsize=(12, 9))
+layers = [s['layer'] for s in layer_stats]
+axes[0,0].bar(layers, [s['mean'] for s in layer_stats]); axes[0,0].axhline(y=off_raw.mean(), color='r', linestyle='--', label=f'raw ({off_raw.mean():.3f})')
+axes[0,0].set_title("Mean cosine"); axes[0,0].legend()
+axes[0,1].bar(layers, [s['near']-s['far'] for s in layer_stats], color='orange'); axes[0,1].axhline(y=near_raw-far_raw, color='r', linestyle='--', label=f'raw delta ({near_raw-far_raw:+.3f})')
+axes[0,1].set_title("Near-Far discriminability"); axes[0,1].legend()
+axes[1,0].bar(layers, [s['gt09'] for s in layer_stats], color='green'); axes[1,0].set_title("# pairs > 0.9")
+axes[1,1].bar(layers, [s['lt_m8'] for s in layer_stats], color='red'); axes[1,1].set_title("# pairs < -0.8")
+fig.suptitle(f"spkw5 post-norm — {Path(AUDIO).stem} ({n} windows)", fontsize=14)
+plt.tight_layout()
+out = Path(__file__).parent / "speaker_spkw5_analysis.png"
+fig.savefig(out, dpi=150, bbox_inches="tight")
+print(f"\nSaved: {out}")
+
+# Heatmaps
+fig2, axes2 = plt.subplots(1, 6, figsize=(22, 3.5))
+im = axes2[0].imshow(sim_raw, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+axes2[0].set_title("Raw (192d)")
+plt.colorbar(im, ax=axes2[0], fraction=0.046)
+plot_layers = [0, 7, 13, 14, 27]
+for col, li in enumerate(plot_layers):
+    ax = axes2[1+col]
+    W = proj[li]; theta = raw @ W.T
+    theta = theta / (np.linalg.norm(theta, axis=1, keepdims=True) + 1e-10)
+    sim = theta @ theta.T
+    im = ax.imshow(sim, cmap="RdBu_r", vmin=-1, vmax=1, aspect="auto")
+    ax.set_title(f"Layer {li} (8d)")
+    plt.colorbar(im, ax=ax, fraction=0.046)
+fig2.suptitle(f"spkw5 — {Path(AUDIO).stem}", fontsize=13)
+plt.tight_layout()
+out2 = Path(__file__).parent / "speaker_spkw5_heatmap.png"
+fig2.savefig(out2, dpi=150, bbox_inches="tight")
+print(f"Saved: {out2}")
+plt.close("all")
diff --git a/MOSS_Speaker-RoPE/wespeaker/.clang-format b/MOSS_Speaker-RoPE/wespeaker/.clang-format
new file mode 100644
index 0000000000000000000000000000000000000000..91dcbc07084b5f2e76391fcaed40dedc87fb267d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.clang-format
@@ -0,0 +1,93 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/MOSS_Speaker-RoPE/wespeaker/.flake8 b/MOSS_Speaker-RoPE/wespeaker/.flake8
new file mode 100644
index 0000000000000000000000000000000000000000..d4e144e81b31498faeb5b66b69a4a9991c40a2d2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.flake8
@@ -0,0 +1,16 @@
+[flake8]
+select = B,C,E,F,P,T4,W,B9
+max-line-length = 80
+max-doc-length = 80
+# C408 ignored because we like the dict keyword argument syntax
+# E501 is not flexible enough, we're using B950 instead
+ignore =
+    E203,E305,E402,E501,E721,E741,F403,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303,
+    # shebang has extra meaning in fbcode lints, so I think it's not worth trying
+    # to line this up with executable bit
+    EXE001,
+    # these ignores are from flake8-bugbear; please fix!
+    B007,B008,B905,
+    # these ignores are from flake8-comprehensions; please fix!
+    C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415
+exclude =
diff --git a/MOSS_Speaker-RoPE/wespeaker/.github/workflows/doc.yml b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/doc.yml
new file mode 100644
index 0000000000000000000000000000000000000000..78b0e5bfe0482ac5542a7421917ba9fe4a61ea1c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/doc.yml
@@ -0,0 +1,55 @@
+name: "Check and Publish Docs"
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'docs/**'
+      - 'wespeaker/**.py'
+  push:
+    branches:
+      - master
+    paths:
+      - 'docs/**'
+      - 'wespeaker/**.py'
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        torch: ["1.13.1"]
+        python-version: ["3.8"]
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build doc
+        shell: bash
+        run: |
+          cd docs
+          python3 -m pip install -r ./requirements.txt
+          make api
+          make html
+
+      # add .nojekyll to notice Pages use the _* dirs
+      - name: copy the generated site
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        run: |
+          mkdir public
+          touch public/.nojekyll
+          cp -r docs/_build/html/* public/
+
+      # push to gh-pages branch
+      - name: github pages deploy
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: peaceiris/actions-gh-pages@v2.3.1
+        env:
+          PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
+          PUBLISH_BRANCH: gh-pages
+          PUBLISH_DIR: public
diff --git a/MOSS_Speaker-RoPE/wespeaker/.github/workflows/lint.yml b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/lint.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d70c248500c83e1a8f8955c1409b3ac9e4e15df6
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/lint.yml
@@ -0,0 +1,87 @@
+name: Lint
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+
+jobs:
+  quick-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch Wenet
+        uses: actions/checkout@v1
+      - name: Checkout PR tip
+        run: |
+          set -eux
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # We are on a PR, so actions/checkout leaves us on a merge commit.
+            # Check out the actual tip of the branch.
+            git checkout ${{ github.event.pull_request.head.sha }}
+          fi
+          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
+        id: get_pr_tip
+      - name: Ensure no tabs
+        run: |
+          (! git grep -I -l $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have tabs; please convert them to spaces"; false))
+      - name: Ensure no trailing whitespace
+        run: |
+          (! git grep -I -n $' $' -- . ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above files have trailing whitespace; please remove them"; false))
+
+  flake8-py3:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.10'
+          architecture: x64
+      - name: Fetch Wenet
+        uses: actions/checkout@v1
+      - name: Checkout PR tip
+        run: |
+          set -eux
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # We are on a PR, so actions/checkout leaves us on a merge commit.
+            # Check out the actual tip of the branch.
+            git checkout ${{ github.event.pull_request.head.sha }}
+          fi
+          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
+        id: get_pr_tip
+      - name: Run flake8
+        run: |
+          set -eux
+          pip install flake8==3.8.2 flake8-bugbear flake8-comprehensions flake8-executable flake8-pyi==20.5.0 mccabe pycodestyle==2.6.0 pyflakes==2.2.0
+          flake8 --version
+          flake8
+          if [ $? != 0 ]; then exit 1; fi
+
+  cpplint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.x
+          architecture: x64
+      - name: Fetch Wenet
+        uses: actions/checkout@v1
+      - name: Checkout PR tip
+        run: |
+          set -eux
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # We are on a PR, so actions/checkout leaves us on a merge commit.
+            # Check out the actual tip of the branch.
+            git checkout ${{ github.event.pull_request.head.sha }}
+          fi
+          echo ::set-output name=commit_sha::$(git rev-parse HEAD)
+        id: get_pr_tip
+      - name: Run cpplint
+        run: |
+          set -eux
+          pip install cpplint==1.6.1
+          cpplint --version
+          cpplint --recursive .
+          if [ $? != 0 ]; then exit 1; fi
+
diff --git a/MOSS_Speaker-RoPE/wespeaker/.github/workflows/runtime.yml b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/runtime.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9f40fbf0ff2f6d73339146e8c3b72ec71af84dfe
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.github/workflows/runtime.yml
@@ -0,0 +1,37 @@
+name: Build Runtime
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'runtime/**'
+
+env:
+  RUNTIME_DIR: runtime/onnxruntime
+  FC_BASE_DIR: runtime/onnxruntime/fc_base
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Cache FC Base
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.FC_BASE_DIR }}
+          key: ${{ runner.os }}-fc-base
+
+      - uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ${{ runner.os }}-build
+
+      - name: Build
+        run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          cd ${{ env.RUNTIME_DIR }}
+          cmake -B build -DCMAKE_BUILD_TYPE=Release
+          cmake --build build -j$(nproc)
diff --git a/MOSS_Speaker-RoPE/wespeaker/.gitignore b/MOSS_Speaker-RoPE/wespeaker/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..aaadf32e747cc6af40a08514e46a8f5ab1fb8642
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.gitignore
@@ -0,0 +1,49 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info
+
+# Visual Studio Code files
+.vscode
+.vs
+
+# PyCharm files
+.idea
+venv
+
+# Eclipse Project settings
+*.*project
+.settings
+
+# Sublime Text settings
+*.sublime-workspace
+*.sublime-project
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*.swm
+*~
+
+# IPython notebook checkpoints
+.ipynb_checkpoints
+
+# macOS dir files
+.DS_Store
+
+exp
+data
+raw_wav
+tensorboard
+**/*build*
+dist/
+*.egg-info/
+*.egg
+
+*.onnx
+external_tools
+pretrained_models
+s3prl_hub
+whisper_hub
diff --git a/MOSS_Speaker-RoPE/wespeaker/.pre-commit-config.yaml b/MOSS_Speaker-RoPE/wespeaker/.pre-commit-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e9e72e0edabfa4edd9e4f2433fde78161d8b7f15
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/.pre-commit-config.yaml
@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    - id: trailing-whitespace
+  - repo: https://github.com/pre-commit/mirrors-yapf
+    rev: 'v0.32.0'
+    hooks:
+    - id: yapf
+      additional_dependencies: [toml]
+  - repo: https://github.com/pycqa/flake8
+    rev: '3.8.2'
+    hooks:
+    - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: 'v17.0.6'
+    hooks:
+    - id: clang-format
+  - repo: https://github.com/cpplint/cpplint
+    rev: '1.6.1'
+    hooks:
+    - id: cpplint
diff --git a/MOSS_Speaker-RoPE/wespeaker/CPPLINT.cfg b/MOSS_Speaker-RoPE/wespeaker/CPPLINT.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..7156af1f2a2216767dba91df1175d9f2d3af79d0
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/CPPLINT.cfg
@@ -0,0 +1,2 @@
+root=runtime/core
+filter=-build/c++11
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/wespeaker/LICENSE b/MOSS_Speaker-RoPE/wespeaker/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/MOSS_Speaker-RoPE/wespeaker/README.md b/MOSS_Speaker-RoPE/wespeaker/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..767584701c7d9a19dbb1392897f3a2087e455874
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/README.md
@@ -0,0 +1,135 @@
+# WeSpeaker
+
+[![License](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Python-Version](https://img.shields.io/badge/Python-3.8%7C3.9-brightgreen)](https://github.com/wenet-e2e/wespeaker)
+
+[**Roadmap**](ROADMAP.md)
+| [**Docs**](http://wenet.org.cn/wespeaker)
+| [**Paper**](https://arxiv.org/abs/2210.17016)
+| [**Runtime**](https://github.com/wenet-e2e/wespeaker/tree/master/runtime)
+| [**Pretrained Models**](docs/pretrained.md)
+| [**Huggingface Demo**](https://huggingface.co/spaces/wenet/wespeaker_demo)
+| [**Modelscope Demo**](https://www.modelscope.cn/studios/wenet/Speaker_Verification_in_WeSpeaker/summary)
+
+
+WeSpeaker mainly focuses on [**speaker embedding learning**](https://wsstriving.github.io/talk/ncmmsc_slides_shuai.pdf), with application to the speaker verification task. We support
+online feature extraction or loading pre-extracted features in kaldi-format.
+
+## Installation
+
+### Install python package
+``` sh
+pip install git+https://github.com/wenet-e2e/wespeaker.git
+```
+**Command-line usage** (use `-h` for parameters):
+
+``` sh
+$ wespeaker --task embedding --audio_file audio.wav --output_file embedding.txt
+$ wespeaker --task embedding_kaldi --wav_scp wav.scp --output_file /path/to/embedding
+$ wespeaker --task similarity --audio_file audio.wav --audio_file2 audio2.wav
+$ wespeaker --task diarization --audio_file audio.wav
+```
+
+**Python programming usage**:
+
+``` python
+import wespeaker
+
+model = wespeaker.load_model('chinese')
+embedding = model.extract_embedding('audio.wav')
+utt_names, embeddings = model.extract_embedding_list('wav.scp')
+similarity = model.compute_similarity('audio1.wav', 'audio2.wav')
+diar_result = model.diarize('audio.wav')
+```
+
+You can set the environment variable `WESPEAKER_HOME` to specify the path of downloaded pre-trained models. By default it will be `$HOME/.wespeaker`.
+
+Please refer to [python usage](docs/python_package.md) for more command line and python programming usage.
+
+### Install for development & deployment
+* Clone this repo
+``` sh
+git clone https://github.com/wenet-e2e/wespeaker.git
+```
+
+* Create conda env: pytorch version >= 2.0.0 is recommended !!!
+``` sh
+conda create -n wespeaker python=3.9
+conda activate wespeaker
+pip install torch==2.7.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu118 # cuda 11.X
+pip install torch==2.7.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu128 # cuda 12.X
+conda install -c conda-forge sox
+pip install -r requirements.txt
+# visdom (dep of torchnet) needs pkg_resources at build time, bypass pip build isolation to use the existing setuptools
+pip install --no-build-isolation torchnet==0.0.4
+pre-commit install  # for clean and tidy code
+```
+
+## 🔥 News
+* 2026.07.03: Integrate the [ReDimNet2](https://arxiv.org/abs/2603.11841) model with the TFMel frontend, see [#465](https://github.com/wenet-e2e/wespeaker/pull/465).
+* 2026.06.15: Add support for [HAM-Softmax loss](https://github.com/PunkMale/HAM-Softmax), see [#464](https://github.com/wenet-e2e/wespeaker/pull/464).
+* 2025.12.05: Add support for the [w2v-bert2 model](https://www.arxiv.org/pdf/2510.04213), see [#439](https://github.com/wenet-e2e/wespeaker/pull/439), [#441](https://github.com/wenet-e2e/wespeaker/pull/441) and [#466](https://github.com/wenet-e2e/wespeaker/pull/466).
+* 2025.02.23: Add support for the Xi-vector, see [#404](https://github.com/wenet-e2e/wespeaker/pull/404).
+* 2024.09.03: Support the SimAM_ResNet and the model pretrained on VoxBlink2, check [Pretrained Models](docs/pretrained.md) for the pretrained model, [VoxCeleb Recipe](https://github.com/wenet-e2e/wespeaker/tree/master/examples/voxceleb/v2) for the super performance, and [python usage](docs/python_package.md) for the command line usage!
+* 2024.08.30: We support whisper_encoder based frontend and propose the [Whisper-PMFA](https://arxiv.org/pdf/2408.15585) framework, check [#356](https://github.com/wenet-e2e/wespeaker/pull/356).
+* 2024.08.20: Update diarization recipe for VoxConverse dataset by leveraging umap dimensionality reduction and hdbscan clustering, see [#347](https://github.com/wenet-e2e/wespeaker/pull/347) and [#352](https://github.com/wenet-e2e/wespeaker/pull/352).
+* 2024.08.18: Support using ssl pre-trained models as the frontend. The [WavLM recipe](https://github.com/wenet-e2e/wespeaker/blob/master/examples/voxceleb/v2/run_wavlm.sh) is also provided, see [#344](https://github.com/wenet-e2e/wespeaker/pull/344).
+* 2024.05.15: Add support for [quality-aware score calibration](https://arxiv.org/pdf/2211.00815), see [#320](https://github.com/wenet-e2e/wespeaker/pull/320).
+* 2024.04.25: Add support for the gemini-dfresnet model, see [#291](https://github.com/wenet-e2e/wespeaker/pull/291).
+* 2024.04.23: Support MNN inference engine in runtime, see [#310](https://github.com/wenet-e2e/wespeaker/pull/310).
+* 2024.04.02: Release [Wespeaker document](http://wenet.org.cn/wespeaker) with detailed model-training tutorials, introduction of various runtime platforms, etc.
+* 2024.03.04: Support the [eres2net-cn-common-200k](https://www.modelscope.cn/models/iic/speech_eres2net_sv_zh-cn_16k-common/summary) and [campplus-cn-common-200k](https://www.modelscope.cn/models/iic/speech_campplus_sv_zh-cn_16k-common/summary) of damo [#281](https://github.com/wenet-e2e/wespeaker/pull/281), check [python usage](https://github.com/wenet-e2e/wespeaker/blob/master/docs/python_package.md) for details.
+* 2024.02.05: Support the ERes2Net [#272](https://github.com/wenet-e2e/wespeaker/pull/272) and Res2Net [#273](https://github.com/wenet-e2e/wespeaker/pull/273) models.
+* 2023.11.13: Support CLI usage of wespeaker, check [python usage](https://github.com/wenet-e2e/wespeaker/blob/master/docs/python_package.md) for details.
+* 2023.07.18: Support the kaldi-compatible PLDA and unsupervised adaptation, see [#186](https://github.com/wenet-e2e/wespeaker/pull/186).
+* 2023.07.14: Support the [NIST SRE16 recipe](https://www.nist.gov/itl/iad/mig/speaker-recognition-evaluation-2016), see [#177](https://github.com/wenet-e2e/wespeaker/pull/177).
+
+## Recipes
+
+* [VoxCeleb](https://github.com/wenet-e2e/wespeaker/tree/master/examples/voxceleb): Speaker Verification recipe on the [VoxCeleb dataset](https://www.robots.ox.ac.uk/~vgg/data/voxceleb/)
+    * 🔥 UPDATE 2024.05.15: We support score calibration for Voxceleb and achieve better performance!
+    * 🔥 UPDATE 2023.07.10: We support self-supervised learning recipe on Voxceleb! Achieving **2.627%** (ECAPA_TDNN_GLOB_c1024) EER on vox1-O-clean test set without any labels.
+    * 🔥 UPDATE 2022.10.31: We support deep r-vector up to the 293-layer version! Achieving **0.447%/0.043** EER/mindcf on vox1-O-clean test set
+    * 🔥 UPDATE 2022.07.19: We apply the same setups as the CNCeleb recipe, and obtain SOTA performance considering the open-source systems
+      - EER/minDCF on vox1-O-clean test set are **0.723%/0.069** (ResNet34) and **0.728%/0.099** (ECAPA_TDNN_GLOB_c1024), after LM fine-tuning and AS-Norm
+* [CNCeleb](https://github.com/wenet-e2e/wespeaker/tree/master/examples/cnceleb/v2): Speaker Verification recipe on the [CnCeleb dataset](http://cnceleb.org/)
+    * 🔥 UPDATE 2024.05.16: We support score calibration for Cnceleb and achieve better EER.
+    * 🔥 UPDATE 2022.10.31: 221-layer ResNet achieves **5.655%/0.330**  EER/minDCF
+    * 🔥 UPDATE 2022.07.12: We migrate the winner system of CNSRC 2022 [report](https://aishell-cnsrc.oss-cn-hangzhou.aliyuncs.com/T082.pdf) [slides](https://aishell-cnsrc.oss-cn-hangzhou.aliyuncs.com/T082-ZhengyangChen.pdf)
+      - EER/minDCF reduction from 8.426%/0.487 to **6.492%/0.354** after large margin fine-tuning and AS-Norm
+* [NIST SRE16](https://github.com/wenet-e2e/wespeaker/tree/master/examples/sre/v2): Speaker Verification recipe for the [2016 NIST Speaker Recognition Evaluation Plan](https://www.nist.gov/itl/iad/mig/speaker-recognition-evaluation-2016). Similar recipe can be found in [Kaldi](https://github.com/kaldi-asr/kaldi/tree/master/egs/sre16).
+   * 🔥 UPDATE 2023.07.14: We support NIST SRE16 recipe. After PLDA adaptation, we achieved 6.608%, 10.01%, and 2.974% EER on trial Pooled, Tagalog, and Cantonese, respectively.
+* [VoxConverse](https://github.com/wenet-e2e/wespeaker/tree/master/examples/voxconverse): Diarization recipe on the [VoxConverse dataset](https://www.robots.ox.ac.uk/~vgg/data/voxconverse/)
+
+## Discussion
+
+For Chinese users, you can scan the QR code on the left to follow our offical account of `WeNet Community`.
+We also created a WeChat group for better discussion and quicker response. Please scan the QR code on the right to join the chat group.
+| <img src="https://github.com/wenet-e2e/wenet-contributors/blob/main/wenet_official.jpeg" width="250px"> | <img src="https://github.com/wenet-e2e/wenet-contributors/blob/main/wespeaker/wangshuai.jpg" width="250px"> |
+| ---- | ---- |
+
+## Citations
+If you find wespeaker useful, please cite it as
+```bibtex
+@article{wang2024advancing,
+  title={Advancing speaker embedding learning: Wespeaker toolkit for research and production},
+  author={Wang, Shuai and Chen, Zhengyang and Han, Bing and Wang, Hongji and Liang, Chengdong and Zhang, Binbin and Xiang, Xu and Ding, Wen and Rohdin, Johan and Silnova, Anna and others},
+  journal={Speech Communication},
+  volume={162},
+  pages={103104},
+  year={2024},
+  publisher={Elsevier}
+}
+
+@inproceedings{wang2023wespeaker,
+  title={Wespeaker: A research and production oriented speaker embedding learning toolkit},
+  author={Wang, Hongji and Liang, Chengdong and Wang, Shuai and Chen, Zhengyang and Zhang, Binbin and Xiang, Xu and Deng, Yanlei and Qian, Yanmin},
+  booktitle={IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={1--5},
+  year={2023},
+  organization={IEEE}
+}
+```
+## Looking for contributors
+
+If you are interested to contribute, feel free to contact @wsstriving or @robin1001
diff --git a/MOSS_Speaker-RoPE/wespeaker/ROADMAP.md b/MOSS_Speaker-RoPE/wespeaker/ROADMAP.md
new file mode 100644
index 0000000000000000000000000000000000000000..2f9aa195c00b814064d0ba4ba7c6ecdc7302c2b3
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/ROADMAP.md
@@ -0,0 +1,112 @@
+# Wespeaker Roadmap
+
+## Version 2.0 (Time: 2023.12)
+
+This is the roadmap for wespeaker version 2.0.
+
+- [ ] SSL support
+    - [ ] Algorithms
+        - [x] DINO
+        - [x] MOCO
+        - [x] SimCLR
+        - [ ] Iteratively psudo label prediction and supervised finetuning
+    - [ ] Recipes
+        - [x] VoxCeleb
+        - [ ] WenetSpeech
+        - [ ] Gigaspeech
+- [ ] Recipes
+    - [ ] 3D-speaker
+    - [ ] NIST SRE
+        - [x] SRE16
+        - [ ] SRE18
+  - [ ] Documents
+    - [ ] Speaker embedding learning basics
+    - [ ] Core code explanation
+    - [x] Step-by-step tutorials
+      - [x] VoxCeleb Supervised
+      - [x] VoxCeleb Self-supervised
+      - [x] VoxSRC Diarization
+
+## Version 1.0 （Time: 2022.09)
+
+This is the roadmap for wespeaker version 1.0.
+
+- [x] Standard dataset support
+    - [x] VoxCeleb
+    - [x] CnCeleb
+- [x] SOTA models support
+    - [x] x-vector (tdnn based, milestone deep speaker embedding)
+    - [x] r-vector (resnet based, winner of voxsrc 2019)
+    - [x] ecapa-tdnn (variant of tdnn, winner of voxsrc 2020)
+- [x] Back-end Support
+    - [x] Cosine
+    - [x] EER/minDCF
+    - [x] AS-norm
+    - [x] PLDA
+- [x] UIO for effective industrial-scale dataset processing
+    - [x] Online data augmentation
+        - Noise && RIR
+        - Speed Perturb
+        - Specaug
+- [x] ONNX support
+- [x] Triton Server support (GPU)
+- [ ] ~~
+    - Training or finetuning big models such as WavLM might be too costly for
+      current stage
+- [x] Basic Speaker Diarization Recipe
+    - Embedding based (more related with our speaker embedding learner toolkit)
+- [x] Interactive Demo
+    - Support using features from released pretrained models (hugging face)
+
+
+## Current Support List
+* Model (SOTA Models)
+    - [x] [Standard X-vector](http://www.danielpovey.com/files/2017_interspeech_embeddings.pdf)
+    - [x] [ResNet](https://arxiv.org/pdf/1512.03385.pdf)
+    - [x] [ECAPA_TDNN](https://arxiv.org/pdf/2005.07143.pdf)
+    - [x] [RepVGG](https://arxiv.org/pdf/2101.03697.pdf)
+    - [x] [CAM++](https://arxiv.org/pdf/2303.00332.pdf)
+    - [x] [ERes2Net](https://arxiv.org/pdf/2305.12838.pdf)
+    - [x] [Gemini-dfresnet](https://arxiv.org/abs/2312.03620)
+* Pooling Functions
+    - [x] TAP(mean) / TSDP(std) / TSTP(mean+std)
+        - Comparison of mean/std pooling can be found in [shuai_iscslp](https://x-lance.sjtu.edu.cn/en/papers/2021/iscslp21_shuai_1_.pdf), [anna_arxiv](https://arxiv.org/pdf/2203.10300.pdf)
+    - [x] Attentive Statistics Pooling (ASTP)
+        - Mainly for ECAPA_TDNN
+    - [x] Multi-Query and Multi-Head Attentive Statistics Pooling (MQMHASTP)
+        - Details can be found in [MQMHASTP](https://arxiv.org/pdf/2110.05042.pdf)
+* Criteria
+    - [x] Softmax
+    - [x] [Sphere (A-Softmax)](https://www.researchgate.net/publication/327389164)
+    - [x] [Add_Margin (AM-Softmax)](https://arxiv.org/pdf/1801.05599.pdf)
+    - [x] [Arc_Margin (AAM-Softmax)](https://arxiv.org/pdf/1801.07698v1.pdf)
+    - [x] [Arc_Margin+Inter-topk+Sub-center](https://arxiv.org/pdf/2110.05042.pdf)
+    - [x] [SphereFace2](https://ieeexplore.ieee.org/abstract/document/10094954)
+* Scoring
+    - [x] Cosine
+    - [x] PLDA
+    - [x] Score Normalization (AS-Norm)
+    - [x] Quality-aware Score Calibration
+* Metric
+    - [x] EER
+    - [x] minDCF
+    - [x] DER
+* Online Augmentation
+    - [x] Noise && RIR
+    - [x] Speed Perturb
+    - [x] SpecAug
+* Training Strategy
+    - [x] Well-designed Learning Rate and Margin Schedulers
+    - [x] Large Margin Fine-tuning
+    - [x] Automatic Mixed Precision (AMP) Training
+* Runtime
+    - [x] Python Binding
+    - [x] Triton Inference Server on verification && diarization in GPU deployment
+    - [x] C++ Onnxruntime
+    - [x] MNN
+* Self-Supervised Learning (SSL)
+    - [x] [DINO](https://openaccess.thecvf.com/content/ICCV2021/papers/Caron_Emerging_Properties_in_Self-Supervised_Vision_Transformers_ICCV_2021_paper.pdf)
+    - [x] [MoCo](https://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf)
+    - [x] [SimCLR](http://proceedings.mlr.press/v119/chen20j/chen20j.pdf)
+* Literature
+    - [x] [Awesome Speaker Papers](docs/speaker_recognition_papers.md)
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/.gitignore b/MOSS_Speaker-RoPE/wespeaker/docs/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..185454ff4d37bb74c10137c8add4dbd75190a036
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/.gitignore
@@ -0,0 +1,3 @@
+_gen/
+_build/
+build/
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/Makefile b/MOSS_Speaker-RoPE/wespeaker/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..53c614a6bb50d93ef67be9abfb3164a38942ef77
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/Makefile
@@ -0,0 +1,26 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SPHINXPROJ    = wespeaker
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+api:
+	sphinx-apidoc -f --separate --module-first -d 2 -o ./python_api ../wespeaker
+	sed -i 's:^wespeaker:Python API Reference:g' ./python_api/modules.rst
+	sed -i 's:^=====:====================:g' ./python_api/modules.rst
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/README.md b/MOSS_Speaker-RoPE/wespeaker/docs/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..789d05288ae9b2448d2790f1ef7beaf9ce8927bf
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/README.md
@@ -0,0 +1,18 @@
+# Basic Documents for WeSpeaker
+
+This directory includes the basic documents for wespeaker, including
+
+- [SV Tutorial on VoxCeleb v2 (Supervised)](https://github.com/wenet-e2e/wespeaker/blob/master/docs/vox.md)
+- [SV Tutorial on VoxCeleb v3 (Self-Supervised-DINO)](https://github.com/wenet-e2e/wespeaker/blob/master/docs/vox_ssl.md)
+- [Diarization Tutorial on VoxConverse v2](https://github.com/wenet-e2e/wespeaker/blob/master/docs/voxconverse_diar.md)
+- [Suggested papers for speaker embedding learning](https://github.com/wenet-e2e/wespeaker/blob/master/docs/speaker_recognition_papers.md)
+- [Provided pretrained models](https://github.com/wenet-e2e/wespeaker/blob/master/docs/pretrained.md)
+- [Off-the-shelf Usages: from Command Line or Python Code](https://github.com/wenet-e2e/wespeaker/blob/master/docs/python_package.md)
+- [How to contribute to Wespeaker](https://github.com/wenet-e2e/wespeaker/blob/master/docs/contribute.md)
+
+- [NCMMSC Tutorial on speaker representation learning \[slides\]](https://wsstriving.github.io/talk/ncmmsc_slides_shuai.pdf)
+
+## ToDo List (possible)
+
+- [ ] Chinese HandBooks
+- [ ] Introduction in Video
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/conf.py b/MOSS_Speaker-RoPE/wespeaker/docs/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..a661ee8d3a9ab994dcb3277750d551543121d520
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/conf.py
@@ -0,0 +1,68 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'wespeaker'
+copyright = '2024, wespeaker-team'
+author = 'wespeaker-team'
+release = '1.2.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "nbsphinx",
+    "sphinx.ext.autodoc",
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    "sphinx.ext.mathjax",
+    "sphinx.ext.todo",
+    # "sphinxarg.ext",
+    "sphinx_markdown_tables",
+    'recommonmark',
+    'sphinx_rtd_theme',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.txt': 'markdown',
+    '.md': 'markdown',
+}
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+# html_theme = 'alabaster'
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/contribute.md b/MOSS_Speaker-RoPE/wespeaker/docs/contribute.md
new file mode 100644
index 0000000000000000000000000000000000000000..552930a1a434972896cd26be0f3bc865538f703f
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/contribute.md
@@ -0,0 +1,54 @@
+## Contributing to Wespeaker
+
+Thank you for your interest in contributing to Wespeaker!
+Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large.
+There are several ways you can contribute to the project:
+
+- Identify and report any issues or bugs.
+- Request or add a new model.
+- Suggest or implement new features.
+
+However, remember that contributions aren't just about code.
+We believe in the power of community support; thus, answering queries, assisting others, and enhancing the documentation
+are highly regarded and beneficial contributions.
+
+Finally, one of the most impactful ways to support us is by raising awareness about Wespeaker.
+Talk about it in your blog posts, highlighting how it's driving your incredible projects.
+
+
+### Issue Reporting
+
+If you encounter a bug or have a feature request, please check our issues page first to see if someone else has already
+reported it.
+If not, please file a new issue, providing as much relevant information as possible.
+
+### Coding Style Guide
+
+In general, we adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html)
+and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
+
+### Pull Requests
+
+When submitting a pull request:
+
+1. Make sure your code has been rebased on top of the latest commit on the master branch.
+2. Ensure code is properly formatted.
+3. Include a detailed description of the changes in the pull request.
+   Explain why you made the changes you did.
+   If your pull request fixes an open issue, please include a reference to it in the description.
+
+### Code Reviews
+
+All submissions, including submissions by project members, require a code review.
+To make the review process as smooth as possible, please:
+
+1. Keep your changes as concise as possible.
+   If your pull request involves multiple unrelated changes, consider splitting it into separate pull requests.
+2. Respond to all comments within a reasonable time frame.
+   If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the
+   suggestion.
+
+### Thank You
+
+Finally, thank you for taking the time to read these guidelines and for your interest in contributing to Wespeaker.
+Your contributions make Wespeaker a great tool for everyone!
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/index.rst b/MOSS_Speaker-RoPE/wespeaker/docs/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3f37923af7f1fd142ca6ff3e1a62bcaa4b4bd9e1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/index.rst
@@ -0,0 +1,27 @@
+.. wespeaker documentation master file, created by
+   sphinx-quickstart on Fri Mar 29 17:16:29 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Wespeaker's documentation!
+=====================================
+
+Wespeaker is an Research and Production Oriented Speaker Verification, Recognition and Diarization Toolkit.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   ./python_package.md
+   ./train.rst
+   ./pretrained.md
+   ./runtime.md
+   ./reference.rst
+   ./contribute.md
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/paper.md b/MOSS_Speaker-RoPE/wespeaker/docs/paper.md
new file mode 100644
index 0000000000000000000000000000000000000000..6e0e0e69d5856538b00e4fa21677c162f302c8c8
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/paper.md
@@ -0,0 +1,4 @@
+# Wespeaker Papers
+
+* [Wespeaker: A research and production oriented speaker embedding learning toolkit](https://arxiv.org/pdf/2210.17016.pdf), accepted by ICASSP 2023.
+* [Wespeaker baselines for VoxSRC2023](https://arxiv.org/pdf/2306.15161.pdf)
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/papers_using_wespeaker.md b/MOSS_Speaker-RoPE/wespeaker/docs/papers_using_wespeaker.md
new file mode 100644
index 0000000000000000000000000000000000000000..2084728be7c01e2509906dab17116ad0c446a5b5
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/papers_using_wespeaker.md
@@ -0,0 +1,227 @@
+# Papers Implemented in WeSpeaker
+
+[TOC]
+
+## Stay Tuned! (Need to add a introduction for each paper)
+
+## Introduction
+
+After the release of the WeSpeaker project, many users from both academia and industry have actively engaged with it in their research. We appreciate all the feedback and contributions from the community and would like to highlight these interesting works.
+
+Besides the citation of WeSpeaker itself, we highly recommend you to read and cite the corresponding papers as listed below.
+
+```bibtex
+@article{wang2024advancing,
+  title={Advancing speaker embedding learning: Wespeaker toolkit for research and production},
+  author={Wang, Shuai and Chen, Zhengyang and Han, Bing and Wang, Hongji and Liang, Chengdong and Zhang, Binbin and Xiang, Xu and Ding, Wen and Rohdin, Johan and Silnova, Anna and others},
+  journal={Speech Communication},
+  volume={162},
+  pages={103104},
+  year={2024},
+  publisher={Elsevier}
+}
+
+@inproceedings{wang2023wespeaker,
+  title={Wespeaker: A research and production oriented speaker embedding learning toolkit},
+  author={Wang, Hongji and Liang, Chengdong and Wang, Shuai and Chen, Zhengyang and Zhang, Binbin and Xiang, Xu and Deng, Yanlei and Qian, Yanmin},
+  booktitle={IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={1--5},
+  year={2023},
+  organization={IEEE}
+}
+```
+
+## Architecture
+
+### TDNN
+
+```bibtex
+@inproceedings{snyder2018x,
+  title={X-vectors: Robust dnn embeddings for speaker recognition},
+  author={Snyder, David and Garcia-Romero, Daniel and Sell, Gregory and Povey, Daniel and Khudanpur, Sanjeev},
+  booktitle={2018 IEEE international conference on acoustics, speech and signal processing (ICASSP)},
+  pages={5329--5333},
+  year={2018},
+  organization={IEEE}
+}
+```
+
+### ECAPA-TDNN
+
+```bibtex
+@article{desplanques2020ecapa,
+  title={Ecapa-tdnn: Emphasized channel attention, propagation and aggregation in tdnn based speaker verification},
+  author={Desplanques, Brecht and Thienpondt, Jenthe and Demuynck, Kris},
+  journal={arXiv preprint arXiv:2005.07143},
+  year={2020}
+}
+```
+
+### Xi-vector
+
+```bibtex
+@article{lee2021xi,
+  title={Xi-vector embedding for speaker recognition},
+  author={Lee, Kong Aik and Wang, Qiongqiong and Koshinaka, Takafumi},
+  journal={IEEE Signal Processing Letters},
+  volume={28},
+  pages={1385--1389},
+  year={2021},
+  publisher={IEEE}
+}
+```
+
+### ResNet
+
+The Current ResNet implementation is based on our system for VoxSRC2019, it's also the default speaker model in Pyannote.audio diarization pipeline (https://huggingface.co/pyannote/wespeaker-voxceleb-resnet34-LM)
+
+```bibtex
+@article{zeinali2019but,
+  title={But system description to voxceleb speaker recognition challenge 2019},
+  author={Zeinali, Hossein and Wang, Shuai and Silnova, Anna and Mat{\v{e}}jka, Pavel and Plchot, Old{\v{r}}ich},
+  journal={arXiv preprint arXiv:1910.12592},
+  year={2019}
+}
+```
+
+### ReDimNet
+
+>
+
+```bibtex
+@article{yakovlev2024reshape,
+  title={Reshape Dimensions Network for Speaker Recognition},
+  author={Yakovlev, Ivan and Makarov, Rostislav and Balykin, Andrei and Malov, Pavel and Okhotnikov, Anton and Torgashov, Nikita},
+  journal={arXiv preprint arXiv:2407.18223},
+  year={2024}
+}
+```
+
+### Golden gemini DF-ResNet
+
+```bibtex
+@article{liu2024golden,
+  title={Golden gemini is all you need: Finding the sweet spots for speaker verification},
+  author={Liu, Tianchi and Lee, Kong Aik and Wang, Qiongqiong and Li, Haizhou},
+  journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing},
+  year={2024},
+  publisher={IEEE}
+}
+```
+
+### SimAM-ResNet
+
+```bibtex
+@inproceedings{qin2022simple,
+  title={Simple attention module based speaker verification with iterative noisy label detection},
+  author={Qin, Xiaoyi and Li, Na and Weng, Chao and Su, Dan and Li, Ming},
+  booktitle={ICASSP 2022-2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={6722--6726},
+  year={2022},
+  organization={IEEE}
+}
+```
+
+### Whisper based Speaker Verification
+
+```bibtex
+@article{zhao2024whisperpmfapartialmultiscalefeature,
+      title={Whisper-PMFA: Partial Multi-Scale Feature Aggregation for Speaker Verification using Whisper Models},
+      author={Yiyang Zhao and Shuai Wang and Guangzhi Sun and Zehua Chen and Chao Zhang and Mingxing Xu and Thomas Fang Zheng},
+      year={2024},
+      eprint={2408.15585},
+      archivePrefix={arXiv},
+      primaryClass={cs.SD},
+      url={https://arxiv.org/abs/2408.15585},
+}
+```
+
+### CAM++
+
+```bibtex
+@article{wang2023cam++,
+  title={Cam++: A fast and efficient network for speaker verification using context-aware masking},
+  author={Wang, Hui and Zheng, Siqi and Chen, Yafeng and Cheng, Luyao and Chen, Qian},
+  journal={arXiv preprint arXiv:2303.00332},
+  year={2023}
+}
+```
+
+### ERes2Net
+
+```bibtex
+@article{chen2023enhanced,
+  title={An enhanced res2net with local and global feature fusion for speaker verification},
+  author={Chen, Yafeng and Zheng, Siqi and Wang, Hui and Cheng, Luyao and Chen, Qian and Qi, Jiajun},
+  journal={arXiv preprint arXiv:2305.12838},
+  year={2023}
+}
+```
+
+## Pipelines
+
+### DINO Pretraining with Large-scale Data
+
+```bibtex
+@inproceedings{wang2024leveraging,
+  title={Leveraging In-the-Wild Data for Effective Self-Supervised Pretraining in Speaker Recognition},
+  author={Wang, Shuai and Bai, Qibing and Liu, Qi and Yu, Jianwei and Chen, Zhengyang and Han, Bing and Qian, Yanmin and Li, Haizhou},
+  booktitle={ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={10901--10905},
+  year={2024},
+  organization={IEEE}
+}
+```
+
+## Dataset
+
+### VoxBlink
+
+```bibtex
+@inproceedings{lin2024voxblink,
+  title={Voxblink: A large scale speaker verification dataset on camera},
+  author={Lin, Yuke and Qin, Xiaoyi and Zhao, Guoqing and Cheng, Ming and Jiang, Ning and Wu, Haiying and Li, Ming},
+  booktitle={ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={10271--10275},
+  year={2024},
+  organization={IEEE}
+}
+
+@article{lin2024voxblink2,
+  title={VoxBlink2: A 100K+ Speaker Recognition Corpus and the Open-Set Speaker-Identification Benchmark},
+  author={Lin, Yuke and Cheng, Ming and Zhang, Fulin and Gao, Yingying and Zhang, Shilei and Li, Ming},
+  journal={arXiv preprint arXiv:2407.11510},
+  year={2024}
+}
+```
+
+### VoxCeleb
+
+```bibtex
+@article{nagrani2017voxceleb,
+  title={Voxceleb: a large-scale speaker identification dataset},
+  author={Nagrani, Arsha and Chung, Joon Son and Zisserman, Andrew},
+  journal={arXiv preprint arXiv:1706.08612},
+  year={2017}
+}
+
+@article{chung2018voxceleb2,
+  title={Voxceleb2: Deep speaker recognition},
+  author={Chung, Joon Son and Nagrani, Arsha and Zisserman, Andrew},
+  journal={arXiv preprint arXiv:1806.05622},
+  year={2018}
+}
+```
+
+### CNCeleb
+
+```bibtex
+@inproceedings{fan2020cn,
+  title={Cn-celeb: a challenging chinese speaker recognition dataset},
+  author={Fan, Yue and Kang, JW and Li, LT and Li, KC and Chen, HL and Cheng, ST and Zhang, PY and Zhou, ZY and Cai, YQ and Wang, Dong},
+  booktitle={ICASSP 2020-2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages={7604--7608},
+  year={2020},
+  organization={IEEE}
+}
+```
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/pretrained.md b/MOSS_Speaker-RoPE/wespeaker/docs/pretrained.md
new file mode 100644
index 0000000000000000000000000000000000000000..102c6ce2caa1aaa0cca47e671e0329878b4cd603
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/pretrained.md
@@ -0,0 +1,76 @@
+# Pretrained Models in Wespeaker
+
+Besides speaker related tasks, speaker embeddings can be utilized for many related tasks which requires speaker
+modeling, such as
+
+- voice conversion
+- text-to-speech
+- speaker adaptive ASR
+- target speaker extraction
+
+For users who would like to verify the SV performance or extract speaker embeddings for the above tasks without
+troubling about training the speaker embedding learner, we provide two types of pretrained models.
+
+1. **Checkpoint Model**, with suffix **.pt**, the model trained and saved as checkpoint by WeSpeaker python code, you can
+   reproduce our published result with it, or you can use it as checkpoint to continue.
+
+2. **Runtime Model**, with suffix **.onnx**, the `runtime model` is exported by `Onnxruntime` on the `checkpoint model`.
+
+## Model License
+
+The pretrained model in WeNet follows the license of it's corresponding dataset.
+For example, the pretrained model on VoxCeleb follows ` Creative Commons Attribution 4.0 International License. `, since
+it is used as license of the VoxCeleb dataset, see https://mm.kaist.ac.kr/datasets/voxceleb/.
+
+## Onnx Inference Demo
+
+To use the pretrained model in `pytorch` format, please directly refer to the `run.sh` in corresponding recipe.
+
+As for extracting speaker embeddings from the `onnx` model, the following is a toy example.
+
+```bash
+# Download the pretrained model in onnx format and save it as onnx_path
+# wav_path is the path to your wave file (16k)
+python wespeaker/bin/infer_onnx.py --onnx_path $onnx_path --wav_path $wav_path
+```
+
+You can easily adapt `infer_onnx.py` to your application, a speaker diarization example can be found
+in [the voxconverse recipe](https://github.com/wenet-e2e/wespeaker/tree/master/examples/voxconverse).
+
+## Model List
+
+The model with suffix **LM** means that it is further fine-tuned using large-margin fine-tuning, which could perform better on long audios, e.g. >3s.
+
+### modelscope
+
+| Datasets                                      | Languages | Checkpoint (pt)                                                                                                                                                                                                                     | Runtime Model (onnx)                                                                                                                                                                                                                  |
+|-----------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet34.zip) / [ResNet34_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet34_LM.zip)     | [ResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet34.onnx) / [ResNet34_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet34_LM.onnx)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet152_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet152_LM.zip)  | [ResNet152_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet152_LM.onnx)      |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet221_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet221_LM.zip)    | [ResNet221_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet221_LM.onnx)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet293_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet293_LM.zip)    | [ResNet293_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_resnet293_LM.onnx)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [CAM++](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_CAM%2B%2B.zip) / [CAM++_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_CAM%2B%2B_LM.zip)                 | [CAM++](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_CAM%2B%2B.onnx) / [CAM++_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_CAM%2B%2B_LM.onnx)          |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ECAPA512](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA512.zip) / [ECAPA512_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA512_LM.zip) / [ECAPA512_DINO](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ecapa512_dino.zip)    | [ECAPA512](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA512.onnx) / [ECAPA512_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA512_LM.onnx)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ECAPA1024](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA1024.zip) / [ECAPA1024_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA1024_LM.zip) | [ECAPA1024](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA1024.onnx) / [ECAPA1024_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_ECAPA1024_LM.onnx) |
+| [VoxCeleb](../examples/voxceleb/v2/README.md)   | EN    | [Gemini_DFResnet114_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_gemini_dfresnet114_LM.zip)| [Gemini_DFResnet114_LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_gemini_dfresnet114_LM.onnx)  |
+| [CNCeleb](../examples/cnceleb/v2/README.md)   | CN        | [ResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=cnceleb_resnet34.zip) / [ResNet34_LM](https://wenet.org.cn/downloads?models=wespeaker&version=cnceleb_resnet34_LM.zip)      | [ResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=cnceleb_resnet34.onnx) / [ResNet34_LM](https://wenet.org.cn/downloads?models=wespeaker&version=cnceleb_resnet34_LM.onnx)         |
+| [VoxBlink2](../examples/voxceleb/v2/README.md) | Multilingual        | [SimAMResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet34.zip)  | [SimAMResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet34.onnx)      |
+| [VoxBlink2 (pretrain) + VoxCeleb2 (finetune)](../examples/voxceleb/v2/README.md) | Multilingual        | [SimAMResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet34_ft.zip)  |[SimAMResNet34](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet34_ft.onnx)  |
+| [VoxBlink2](../examples/voxceleb/v2/README.md) | Multilingual        | [SimAMResNet100](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet100.zip)  |[SimAMResNet100](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet100.onnx)|
+| [VoxBlink2 (pretrain) + VoxCeleb2 (finetune)](../examples/voxceleb/v2/README.md) | Multilingual        | [SimAMResNet100](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet100_ft.zip)  |[SimAMResNet100](https://wenet.org.cn/downloads?models=wespeaker&version=voxblink2_samresnet100_ft.onnx) |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN | [W2V-BERT2.0](https://www.modelscope.cn/models/shangguanqituan/wespeaker-w2v-bert2/resolve/master/w2v_bert2_voxceleb_reproduced.pt) / [W2V-BERT2.0_LM](https://www.modelscope.cn/models/shangguanqituan/wespeaker-w2v-bert2/resolve/master/w2v_bert2_voxceleb_reproduced_LM.pt) | - |
+| [VoxCeleb + VoxBlink2](../examples/voxceleb/v2/README.md) [paper link](https://www.arxiv.org/pdf/2510.04213) | EN | [W2V-BERT2.0-MFA](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_voxblink2_w2v_bert2_lora_adapterMFA.tar.gz) / [W2V-BERT2.0-MFA-LM](https://wenet.org.cn/downloads?models=wespeaker&version=voxceleb_voxblink2_w2v_bert2_lora_adapterMFA_lm.tar.gz) | - |
+### huggingface
+
+| Datasets                                      | Languages | Checkpoint (pt)                                                                                                                                                                                                                     | Runtime Model (onnx)                                                                                                                                                                                                                  |
+|-----------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet34](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet34/tree/main) / [ResNet34_LM](https://huggingface.co/Wespeaker/wespeaker-resnet34-LM/tree/main)     | [ResNet34](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet34/resolve/main/voxceleb_resnet34.onnx?download=true) / [ResNet34_LM](https://huggingface.co/Wespeaker/wespeaker-resnet34-LM/resolve/main/voxceleb_resnet34_LM.onnx?download=true)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet152_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet152-LM/tree/main)  | [ResNet152_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet152-LM/resolve/main/voxceleb_resnet152_LM.onnx?download=true)      |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet221_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet221-LM/tree/main)    | [ResNet221_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet221-LM/resolve/main/voxceleb_resnet221_LM.onnx?download=true)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ResNet293_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet293-LM/tree/main)    | [ResNet293_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet293-LM/resolve/main/voxceleb_resnet293_LM.onnx?download=true)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [CAM++](https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus/tree/main) / [CAM++_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus-LM/tree/main)                 | [CAM++](https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus/resolve/main/voxceleb_CAM%2B%2B.onnx?download=true) / [CAM++_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-campplus-LM/resolve/main/voxceleb_CAM%2B%2B_LM.onnx?download=true)          |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ECAPA512](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn512/tree/main) / [ECAPA512_LM](https://huggingface.co/Wespeaker/wespeaker-ecapa-tdnn512-LM/tree/main)     | [ECAPA512](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn512/resolve/main/voxceleb_ECAPA512.onnx?download=true) / [ECAPA512_LM](https://huggingface.co/Wespeaker/wespeaker-ecapa-tdnn512-LM/resolve/main/voxceleb_ECAPA512_LM.onnx?download=true)     |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ECAPA1024](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn1024/tree/main) / [ECAPA1024_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn1024-LM/tree/main) | [ECAPA1024](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn1024/resolve/main/voxceleb_ECAPA1024.onnx?download=true) / [ECAPA1024_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-ecapa-tdnn1024-LM/resolve/main/voxceleb_ECAPA1024_LM.onnx?download=true) |
+| [VoxCeleb](../examples/voxceleb/v2/README.md)   | EN    | [Gemini_DFResnet114_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-gemini-DFresnet114-LM/tree/main)| [Gemini_DFResnet114_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-gemini-DFresnet114-LM/resolve/main/voxceleb_gemini_dfresnet114_LM.onnx?download=true)  |
+| [CNCeleb](../examples/cnceleb/v2/README.md)   | CN        | [ResNet34](https://huggingface.co/Wespeaker/wespeaker-cnceleb-resnet34/tree/main) / [ResNet34_LM](https://huggingface.co/Wespeaker/wespeaker-cnceleb-resnet34-LM/tree/main)         | [ResNet34](https://huggingface.co/Wespeaker/wespeaker-cnceleb-resnet34/resolve/main/cnceleb_resnet34.onnx?download=true) / [ResNet34_LM](https://huggingface.co/Wespeaker/wespeaker-cnceleb-resnet34-LM/resolve/main/cnceleb_resnet34_LM.onnx?download=true)         |
+| [VoxCeleb](../examples/voxceleb/v2/README.md) | EN        | [ReDimNet2B6_LM](https://huggingface.co/Wespeaker/wespeaker-voxceleb-redimnet2-B6-LM/tree/main) | - |
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/python_package.md b/MOSS_Speaker-RoPE/wespeaker/docs/python_package.md
new file mode 100644
index 0000000000000000000000000000000000000000..48ed851b74e11c8aa2aa55a96b4b663980d13c4d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/python_package.md
@@ -0,0 +1,92 @@
+# Python Package
+
+## Install
+
+``` sh
+pip install git+https://github.com/wenet-e2e/wespeaker.git
+```
+
+for development install:
+
+``` sh
+git clone https://github.com/wenet-e2e/wespeaker.git
+cd wespeaker
+pip install -e .
+```
+
+## Command Line Usage
+
+``` sh
+$ wespeaker --task embedding --audio_file audio.wav --output_file embedding.txt
+$ wespeaker --task embedding_kaldi --wav_scp wav.scp --output_file /path/to/embedding
+$ wespeaker --task similarity --audio_file audio.wav --audio_file2 audio2.wav
+$ wespeaker --task diarization --audio_file audio.wav
+$ wespeaker --task diarization --audio_file audio.wav --device cuda:0 # use CUDA on Windows/Linux
+$ wespeaker --task diarization --audio_file audio.wav --device mps    # use Metal Performance Shaders on MacOS
+```
+
+You can specify the following parameters. (use `-h` for details)
+
+* `-t` or `--task`: five tasks are supported now
+    - embedding: extract embedding for an audio and save it into an output file
+    - embedding_kaldi: extract embeddings from kaldi-style wav.scp and save it to ark/scp files.
+    - similarity: compute similarity of two audios (in the range of [0, 1])
+    - diarization: apply speaker diarization for an input audio
+    - diarization_list: apply speaker diarization for a kaldi-style wav.scp
+* `-l` or `--language`: use Chinese/English speaker models
+* `-p` or `--pretrain`: the path of pretrained model, `avg_model.pt` and `config.yaml` should be contained
+* `--device`: set pytorch device, `cpu`, `cuda`, `cuda:0` or `mps`
+* `--campplus`:
+  use [`campplus_cn_common_200k` of damo](https://www.modelscope.cn/models/iic/speech_campplus_sv_zh-cn_16k-common/summary)
+* `--eres2net`:
+  use [`res2net_cn_common_200k` of damo](https://www.modelscope.cn/models/iic/speech_eres2net_sv_zh-cn_16k-common/summary)
+* `--vblinkp`: use the sam_resnet34 model pretrained on VoxBlink2
+* `--vblinkf`: use the sam_resnet34 model pretrained on VoxBlink2 and finetuned on VoxCeleb2
+* `--w2vbert2_mfa`: use the w2v-bert2 model pretrained, [paper link](https://www.arxiv.org/pdf/2510.04213)
+* `--audio_file`: input audio file path
+* `--audio_file2`: input audio file2 path, specifically for the similarity task
+* `--wav_scp`: input wav.scp file in kaldi format (each line: key wav_path)
+* `--resample_rate`: resample rate (default: 16000)
+* `--vad`: apply vad or not for the input audios (default: true)
+* `--output_file`: output file to save speaker embedding, if you use kaldi wav_scp, output will be `output_file.ark`
+                   and `output_file.scp`
+
+### Pretrained model support
+
+We provide different pretrained models, which can be found
+at [pretrained models](https://github.com/wenet-e2e/wespeaker/blob/master/docs/pretrained.md).
+
+**Warning** If you want to use the models provided in the above link, be sure to rename the model and config file
+to `avg_model.pt` and `config.yaml`.
+
+By default, specifying the `language` option will download the pretrained models as
+
+* english: `ResNet221_LM` pretrained on VoxCeleb
+* chinese: `ResNet34_LM` pretrained on CnCeleb
+
+If you want to use other pretrained models, please use the `-p` or `--pretrain` to specify the directory
+containing `avg_model.pt` and `config.yaml`,
+which can either be the ones we provided and trained by yourself.
+
+## Python Programming Usage
+
+``` python
+import wespeaker
+
+model = wespeaker.load_model('chinese')
+# set the device on which tensors are or will be allocated.
+model.set_device('cuda:0')
+
+# embedding/embedding_kaldi/similarity/diarization
+embedding = model.extract_embedding('audio.wav')
+utt_names, embeddings = model.extract_embedding_list('wav.scp')
+similarity = model.compute_similarity('audio1.wav', 'audio2.wav')
+diar_result = model.diarize('audio.wav', 'give_this_utt_a_name')
+
+# register and recognize
+model.register('spk1', 'spk1_audio1.wav')
+model.register('spk2', 'spk2_audio1.wav')
+model.register('spk3', 'spk3_audio1.wav')
+result = model.recognize('spk1_audio2.wav')
+```
+
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/reference.rst b/MOSS_Speaker-RoPE/wespeaker/docs/reference.rst
new file mode 100644
index 0000000000000000000000000000000000000000..76806108bb15c5e49bcfb54f52c226cd0269d125
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/reference.rst
@@ -0,0 +1,11 @@
+Reference
+=========
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contents:
+
+   ./paper.md
+   ./speaker_recognition_papers.md
+   ./papers_using_wespeaker.md
+   ./python_api/modules.rst
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/requirements.txt b/MOSS_Speaker-RoPE/wespeaker/docs/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..352ded13f263ba59e7489c22397efe33f67fc503
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/requirements.txt
@@ -0,0 +1,6 @@
+jinja2
+nbsphinx
+sphinx
+recommonmark
+sphinx-markdown-tables
+sphinx-rtd-theme
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/runtime.md b/MOSS_Speaker-RoPE/wespeaker/docs/runtime.md
new file mode 100644
index 0000000000000000000000000000000000000000..79ce4f6b7bf45e4a4371c723b5b0887a335134c2
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/runtime.md
@@ -0,0 +1,249 @@
+# Runtime for Wespeaker
+
+## Platforms Supported
+
+The Wespeaker runtime supports the following platforms.
+
+- Server
+    - [TensorRT GPU](https://github.com/wenet-e2e/wespeaker/tree/master/runtime/server/x86_gpu)
+
+- Device
+    - [Onnxruntime](https://github.com/wenet-e2e/wespeaker/tree/master/runtime/onnxruntime)
+        - linux_x86_cpu
+        - linux_x86_gpu
+        - macOS
+        - windows
+    - Android (coming)
+    - ncnn (coming)
+
+## Onnxruntime
+
+* Step 1. Export your experiment model to ONNX by https://github.com/wenet-e2e/wespeaker/blob/master/wespeaker/bin/export_onnx.py
+
+``` sh
+exp=exp  # Change it to your experiment dir
+onnx_dir=onnx
+python wespeaker/bin/export_onnx.py \
+  --config $exp/config.yaml \
+  --checkpoint $exp/avg_model.pt \
+  --output_model $onnx_dir/final.onnx
+
+# When it finishes, you can find `final.onnx`.
+```
+
+* Step 2. Build. The build requires cmake 3.14 or above, and gcc/g++ 5.4 or above.
+
+``` sh
+mkdir build && cd build
+# 1. no gpu
+cmake -DONNX=ON ..
+# 2. gpu (macOS don't supported)
+# cmake -DONNX=ON -DGPU=ON ..
+cmake --build .
+```
+
+* Step 3. Testing.
+
+> NOTE: If using GPU, you need to specify the cuda path.
+> ```bash
+> export PATH=/usr/local/cuda-11.1/bin${PATH:+:${PATH}}
+> export LD_LIBRARY_PATH=/usr/local/cuda-11.1/lib64:${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+> ```
+
+1. the RTF(real time factor) is shown in the console, and embedding will be written to the txt file.
+``` sh
+export GLOG_logtostderr=1
+export GLOG_v=2
+wav_scp=your_test_wav_scp
+onnx_dir=your_model_dir
+embed_out=your_embedding_txt
+./build/bin/extract_emb_main \
+  --wav_scp $wav_scp \
+  --result $embed_out \
+  --speaker_model_path $onnx_dir/final.onnx \
+  --embedding_size 256 \
+  --samples_per_chunk  32000  # 2s
+
+```
+
+> NOTE: samples_per_chunk: samples of one chunk. samples_per_chunk = sample_rate * duration
+>
+> If samples_per_chunk = -1, compute the embedding of whole sentence;
+> else compute embedding with chunk by chunk, and then average embeddings of chunk.
+
+2. Calculate the similarity of two speech.
+```sh
+export GLOG_logtostderr=1
+export GLOG_v=2
+onnx_dir=your_model_dir
+./build/bin/asv_main \
+    --enroll_wav wav1_path \
+    --test_wav wav2_path \
+    --threshold 0.5 \
+    --speaker_model_path $onnx_dir/final.onnx \
+    --embedding_size 256
+```
+
+## Server (tensorrt gpu)
+
+### Introduction
+In this project, we use models trained in [wespeaker](https://github.com/wenet-e2e/wespeaker) as an example to show how to convert speaker model to tensorrt and deploy them on [Triton Inference Server](https://github.com/triton-inference-server/server.git). If you only have CPUs, instead of using GPUs to deploy Tensorrt model, you may deploy the exported onnx model on Triton Inference Server as well.
+
+### Step 0. Train a model
+Please follow wespeaker examples to train a model. After training, you should get several checkpoints under your `exp/xxx/models/` folder. We take [voxceleb](https://github.com/wenet-e2e/wespeaker/tree/master/examples/voxceleb/v2) as an example.
+
+### Step 1. Export model
+We'll first export our model to onnx and then convert our onnx model to tensorrt.
+```
+# go to your example
+cd wespeaker/examples/voxceleb/v2
+. ./path.sh
+exp_dir=exp/resnet
+python3 wespeaker/bin/export_onnx.py --config=${exp_dir}/config.yaml --checkpoint=${exp_dir}/models/avg_model.pt --output_model=${exp_dir}/models/avg_model.onnx
+
+# If you want to minus the mean vector in the onnx model, you may simply add the --mean_vec to the .npy mean vector file.
+python3 wespeaker/bin/export_onnx.py --config=${exp_dir}/config.yaml --checkpoint=exp/resnet/models/avg_model.pt --output_model=exp/resnet/models/avg_model.onnx --mean_vec=${exp_dir}/embeddings/vox2_dev/mean_vec.npy
+```
+
+If you only want to deploy the onnx model on CPU or GPU, you may skip the Tensorrt part and go to [the section](#construct-model-repo) to construct your model repository.
+
+#### Export to Tensorrt Engine
+Now let's convert our onnx model to tensorrt engine. We will deploy our model on Triton 22.03 therefore we here will use tensorrt 22.03 docker as an example to show how to convert the model. Please move your onnx model to the target platform/GPU you will deploy.
+
+```
+docker run --gpus '"device=0"' -it -v <the output onnx model directory>:/models nvcr.io/nvidia/tensorrt:22.03-py3
+cd /models/
+# shape=BxTxF  batchsize, sequence_length, feature_size
+trtexec --saveEngine=b1_b128_s3000_fp16.trt  --onnx=/models/avg_model.onnx --minShapes=feats:1x200x80 --optShapes=feats:64x200x80 --maxShapes=feats:128x3000x80 --fp16
+```
+Here we get an engine which has maximum sequence length of 3000 and minimum length of 200. Since the frame stride is 10ms, 200 and 3000 corresponds to 2.02 seconds and 30.02 seconds respectively(kaldi feature extractor). Notice these numbers will differ and depend on your feature extractor parameters. Notice we've added `--fp16` and in pratice, we found this option will not affect the final accuracy and improve the perf at the same time.
+
+You may set these numbers by your production requirements. If you only know the seconds of audio you will use and have no idea of how many frames it will generate, you may try the below script:
+```python
+import torchaudio.compliance.kaldi as kaldi
+import torch
+audio_dur_in_seconds = 2
+feat_dim = 80  # please check config.yaml if you dont know
+sample_rate = 16000
+
+waveform = torch.ones(sample_rate * audio_dur_in_seconds).unsqueeze(0)
+feat_tensor = kaldi.fbank(waveform,
+                            num_mel_bins=feat_dim,
+                            frame_shift=10,
+                            frame_length=25,
+                            energy_floor=0.0,
+                            window_type='hamming',
+                            htk_compat=True,
+                            use_energy=False,
+                            dither=1)
+print(feat_tensor.shape) # (198, 80)
+```
+Then you may find `198` is the actual number of frames for audio of 2 seconds long.
+
+That's it！We build an engine that can accept 2.02 to 30.02 seconds long audio. If your application can accept fixed audio segments, we suggest you to set the `minShapes`, `optShapes` and `maxShapes` to the same shape.
+
+#### Construct Model Repo
+
+Now edit the config file under `model_repo/speaker_model/config.pbtxt` and replace `default_model_filename:xxx` with the name of your engine (e.g., `b1_b128_s3000_fp16.trt`) or onnx model (e.g., `avg_model.onnx`) and put the engine or model under `model_repo/speaker_model/1/`.
+
+And if you use other model settings or different model from ours (resnet34), for example, ecapa model, the embedding dim of which is 192, therefore, you should edit the `model_repo/speaker_model/config.pbtxt` and `model_repo/speaker/config.pbtxt` and set embedding dim to 192.
+
+If your model is onnx model, you should also edit `backend: "tensorrt"` to `backend: "onnxruntime"` in `model_repo/speaker_model/config.pbtxt`.
+
+If you want to deploy model on CPUs, you should edit `config.pbtxt` under `speaker` and `speaker_model` and replace `kind: KIND_GPU` to `kind: KIND_CPU`.
+
+Notice Tensorrt engine can only run on GPUs.
+
+### Step 2. Build server and start server
+
+Notice we use triton 22.03 in dockerfile. Be sure to use the triton that has the same version as your tensorrt.
+
+Build server:
+```
+# server
+docker build . -f Dockerfile/dockerfile.server -t wespeaker:latest --network host
+```
+
+```
+docker run --gpus '"device=0"' -v $PWD/model_repo:/ws/model_repo --shm-size=1g --ulimit memlock=-1 -p 8000:8000 -p 8001:8001 -p 8002:8002 --ulimit stack=67108864 -ti  wespeaker:latest
+tritonserver --model-repository=/ws/model_repo
+```
+The port `8000` is for http request and `8001` for grpc request.
+
+### Step 3. Build client and start client
+
+Build client:
+
+```
+# client
+docker build . -f Dockerfile/dockerfile.client -t wespeaker_client:latest --network host
+```
+
+```
+docker run -it -v $PWD:/ws -v <data path>:<data path> --network=host wespeaker_client
+
+# example command
+cd /ws/client/
+python3 client.py --url=<ip of the server>:8001 --wavscp=/raid/dgxsa/slyne/wespeaker/examples/voxceleb/v2/data/vox1/wav.scp --output_directory=<to put the generated embeddings>
+
+# The output direcotry will be something like:
+# xvector_000.ark xvextor_000.scp xvector_001.scp .....
+
+```
+
+### Step 4. Test score
+After you extract the embeddings, you can now use the same way as wespeaker to test these embeddings. For example, you may test the extracted embeddings in wespeaker by:
+```
+cat embeddings/xvector_*.scp > embeddings/xvector.scp
+
+config=conf/resnet.yaml
+exp_dir=exp/resnet
+
+mkdir -p embeddings/scores
+trials_dir=data/vox1/trials
+python -u wespeaker/bin/score.py \
+    --exp_dir ${exp_dir} \
+    --eval_scp_path /raid/dgxsa/slyne/wespeaker/runtime/server/x86_gpu/embeddings/xvector.scp \  # embeddings generated from our server
+    --cal_mean True \
+    --cal_mean_dir ${exp_dir}/embeddings/vox2_dev \
+    --p_target 0.01 \
+    --c_miss 1 \
+    --c_fa 1 \
+    ${trials_dir}/vox1_O_cleaned.kaldi ${trials_dir}/vox1_E_cleaned.kaldi ${trials_dir}/vox1_H_cleaned.kaldi \
+    2>&1 | tee /raid/dgxsa/slyne/wespeaker/runtime/server/x86_gpu/embeddings/scores/vox1_cos_result
+```
+
+### Perf
+
+We build our engines for 2.02 seconds long audio only by:
+```
+trtexec --saveEngine=resnet_b1_b128_s200_fp16.trt  --onnx=resnet/resnet_avg_model.onnx --minShapes=feats:1x200x80 --optShapes=feats:64x200x80 --maxShapes=feats:128x200x80 --fp16
+
+trtexec --saveEngine=ecapa_b1_b128_s200_fp16.trt  --onnx=ecapa/ecapa_avg_model.onnx --minShapes=feats:1x200x80 --optShapes=feats:64x200x80 --maxShapes=feats:128x200x80 --fp16
+```
+
+* GPU: T4
+* resnet: resnet34.
+
+|Engine                              |Throughput (bz=64)| utter/s|
+|------------------------------------|------------------|--------|
+|resnet_b1_b128_s200_fp16.trt        |39.7842           |2546    |
+|ecapa_b1_b128_s200_fp16.trt         |52.958            |3389    |
+
+#### Pipeline Perf
+
+In client docker, we may test the whole pipeline performance.
+```
+cd client/
+# generate test input
+python3 generate_input.py --audio_file=test.wav --seconds=2.02
+
+perf_analyzer -m speaker -b 1 --concurrency-range 200:1000:200 --input-data=input.json -u localhost:8000
+```
+
+|Engine                            | Conccurency | Throughput | Avg Latency(ms) | P99 Latency(ms) |
+|----------------------------------|-------------|------------|-----------------|-----------------|
+|resnet_b1_b128_s200_fp16.trt      | 200         | 2033       | 98              | 111             |
+|                                  | 400         | 2010       | 198             | 208             |
+|ecapa_b1_b128_s200_fp16.trt       | 200         | 2647       | 75              | 111             |
+|                                  | 400         | 2726       | 147             | 172             |
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/speaker_recognition_papers.md b/MOSS_Speaker-RoPE/wespeaker/docs/speaker_recognition_papers.md
new file mode 100644
index 0000000000000000000000000000000000000000..cabe1a4ce6f6b551f9543a5fb545c14358de18dd
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/speaker_recognition_papers.md
@@ -0,0 +1,40 @@
+# Speaker Recognition Papers
+
+- Dataset
+    - VoxCeleb
+        - [Voxceleb: A Large-Scale Speaker Identification Dataset](https://www.robots.ox.ac.uk/~vgg/publications/2017/Nagrani17/nagrani17.pdf)
+        - [Voxceleb2: Deep Speaker Recognition](https://www.robots.ox.ac.uk/~vgg/publications/2018/Chung18a/chung18a.pdf)
+        - [Voxceleb: Large-Scale Speaker Verification In The Wild](https://www.robots.ox.ac.uk/~vgg/publications/2019/Nagrani19/nagrani19.pdf)
+    - CnCeleb
+        - [Cn-Celeb: A Challenging Chinese Speaker Recognition Dataset](https://arxiv.org/pdf/1911.01799.pdf)
+        - [Cn-Celeb: Multi-Genre Speaker Recognition](https://arxiv.org/pdf/2012.12468.pdf)
+- Architecture Design
+    - [X-Vectors: Robust Dnn Embeddings For Speaker Recognition](https://www.danielpovey.com/files/2018_icassp_xvectors.pdf) (
+      **x-vector**)
+    - [But System Description To Voxceleb Speaker Recognition Challenge 2019](https://arxiv.org/pdf/1910.12592.pdf) (*
+      *r-vector**)
+    - [Rawnet: Advanced End-To-End Deep Neural Network Using Raw Waveforms For Text-Independent Speaker Verification](https://arxiv.org/pdf/1904.08104.pdf) (
+      **RawNet**)
+    - [Speaker Recognition From Raw Waveform With Sincnet](https://arxiv.org/pdf/1808.00158.pdf) (**SincNet**)
+    - [Ecapa-Tdnn: Emphasized Channel Attention, Propagation And Aggregation In Tdnn Based Speaker Verification](https://arxiv.org/pdf/2005.07143.pdf) (
+      **ECAPA-TDNN**)
+- Optimization Objective
+    - Classification Based Loss
+        - [Exploring The Encoding Layer And Loss Function In End-To-End Speaker And Language Recognition System](https://arxiv.org/pdf/1804.05160.pdf)
+        - [Angular Softmax For Short-Duration Text-Independent Speaker Verification](https://www.researchgate.net/publication/327389164)
+        - [Ensemble Additive Margin Softmax For Speaker Verification](https://cs.nju.edu.cn/lwj/paper/ICASSP19_EAMS.pdf)
+        - [Margin Matters: Towards More Discriminative Deep Neural Network Embeddings For Speaker Recognition](https://arxiv.org/pdf/1906.07317.pdf)
+        - [Large Margin Softmax Loss For Speaker Verification](https://arxiv.org/abs/1904.03479)
+    - End-to-End Loss
+        - [End-To-End Text-Dependent Speaker Verification](https://arxiv.org/pdf/1509.08062.pdf)
+        - [End-To-End Text-Independent Speaker Verification With Triplet Loss On Short Utterances](https://www.researchgate.net/publication/317416159)
+        - [Generalized End-To-End Loss For Speaker Verification](https://arxiv.org/pdf/1710.10467.pdf)
+- Pooling Method
+    - [Attentive Statistics Pooling For Deep Speaker Embedding](https://arxiv.org/pdf/1803.10963.pdf)
+    - [Multi-Resolution Multi-Head Attention In Deep Speaker Embedding](https://ieeexplore.ieee.org/abstract/document/9053217)
+    - [Utterance-Level Aggregation For Speaker Recognition In The Wild](https://ieeexplore.ieee.org/abstract/document/8683120)
+    - [A Novel Learnable Dictionary Encoding Layer For End-To-End Language Identification](https://arxiv.org/abs/1804.00385)
+- Self-supervised Learning
+    - [Augmentation Adversarial Training For Self-Supervised Speaker Recognition](https://arxiv.org/pdf/2007.12085.pdf)
+    - [Self-Supervised Text-Independent Speaker Verification Using Prototypical Momentum Contrastive Learning](https://arxiv.org/pdf/2012.07178.pdf)
+    - [Self-Supervised Speaker Recognition With Loss-Gated Learning](https://arxiv.org/pdf/2110.03869.pdf)
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/train.rst b/MOSS_Speaker-RoPE/wespeaker/docs/train.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4eea8f26a27c5c9e7a148d456069cea3aaf0e1cc
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/train.rst
@@ -0,0 +1,10 @@
+How to train models?
+====================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contents:
+
+   ./vox.md
+   ./vox_ssl.md
+   ./voxconverse_diar.md
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/vox.md b/MOSS_Speaker-RoPE/wespeaker/docs/vox.md
new file mode 100644
index 0000000000000000000000000000000000000000..8702669ba1baa4d51da688d2d864de8ee3209229
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/vox.md
@@ -0,0 +1,298 @@
+## SV Tutorial on VoxCeleb v2 (Supervised)
+
+If you meet any problems when going through this tutorial, please feel free to ask in
+github [issues](https://github.com/wenet-e2e/wespeaker/issues). Thanks for any kind of feedback.
+
+### First Experiment
+
+We provide a recipe `examples/voxceleb/v2/run.sh` on voxceleb data.
+
+The recipe is simple and we suggest you run each stage one by one manually and check the result to understand the whole
+process.
+
+```
+cd examples/voxceleb/v2/
+bash run.sh --stage 1 --stop_stage 1
+bash run.sh --stage 2 --stop_stage 2
+bash run.sh --stage 3 --stop_stage 3
+bash run.sh --stage 4 --stop_stage 4
+bash run.sh --stage 5 --stop_stage 5
+bash run.sh --stage 6 --stop_stage 6
+bash run.sh --stage 7 --stop_stage 7
+bash run.sh --stage 8 --stop_stage 8
+```
+
+### Stage 1: Download Data
+
+```
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "Prepare datasets ..."
+  ./local/prepare_data.sh --stage 2 --stop_stage 4 --data ${data}
+fi
+```
+
+This stage prepares the **voxceleb1**, **voxceleb2**, **MUSAN** and **RIRS_NOISES** dataset. MUSAN is a noise dataset
+and RIRS_NOISES is a reverberation dataset, which are used for data augmentation.
+
+It should be noted that the `./local/prepare_data.sh` script starts from the stage 2. It is because the data downloading
+process in stage 1 will take a long time. Thus we recommand you to download all archives above in your own way first and
+put it under `data/download_data` and then run the above script.
+
+When finishing this stage, you will get the following meta files:
+
+* **wav.scp** files for all the dataset:
+    * `data/musan/wav.scp`
+    * `data/rirs/wav.scp`
+    * `data/vox1/wav.scp`
+    * `data/vox2_dev/wav.scp`
+* **utt2spk** and **spk2utt** files for voxceleb1 and voxceleb2_dev
+    * `data/vox1/utt2spk`
+    * `data/vox1/spk2utt`
+    * `data/vox2_dev/utt2spk`
+    * `data/vox2_dev/spk2utt`
+* **trials**
+    * `data/vox1/trials/vox1_O_cleaned.kaldi`
+    * `data/vox1/trials/vox1_E_cleaned.kaldi`
+    * `data/vox1/trials/vox1_H_cleaned.kaldi`
+
+**wav.scp** each line records two blank-separated columns : `wav_id` and `wav_path`
+
+```
+id10001/1zcIwhmdeo4/00001.wav /exported/data/voxceleb1_wav_v2/id10001/1zcIwhmdeo4/00001.wav
+id10001/1zcIwhmdeo4/00002.wav /exported/data/voxceleb1_wav_v2/id10001/1zcIwhmdeo4/00002.wav
+...
+```
+
+**utt2spk** each line records two blank-separated columns : `wav_id` and `spk_id`
+
+```
+id10001/1zcIwhmdeo4/00001.wav id10001
+id10001/1zcIwhmdeo4/00002.wav id10001
+...
+```
+
+**spk2utt** each line records many blank-separated columns : `spk_id` and many `wav_id`s belong to this `spk_id`
+
+```
+id10001 id10001/1zcIwhmdeo4/00001.wav id10001/1zcIwhmdeo4/00002.wav id10001/1zcIwhmdeo4/00003.wav ...
+id10002 id10002/0_laIeN-Q44/00001.wav id10002/6WO410QOeuo/00001.wav ...
+...
+```
+
+**trials** each line records three blank-separated columns : `enroll_wav_id`, `test_wav_id` and `label`
+
+```
+id10001/Y8hIVOBuels/00001.wav id10001/1zcIwhmdeo4/00001.wav target
+id10001/Y8hIVOBuels/00001.wav id10943/vNCVj7yLWPU/00005.wav nontarget
+id10001/Y8hIVOBuels/00001.wav id10001/7w0IBEWc9Qw/00004.wav target
+id10001/Y8hIVOBuels/00001.wav id10999/G5R2-Hl7YX8/00008.wav nontarget
+...
+```
+
+### Stage 2: Reformat the Data
+
+```
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "Covert train and test data to ${data_type}..."
+  for dset in vox2_dev vox1; do
+    if [ $data_type == "shard" ]; then
+      python tools/make_shard_list.py --num_utts_per_shard 1000 \
+          --num_threads 16 \
+          --prefix shards \
+          --shuffle \
+          ${data}/$dset/wav.scp ${data}/$dset/utt2spk \
+          ${data}/$dset/shards ${data}/$dset/shard.list
+    else
+      python tools/make_raw_list.py ${data}/$dset/wav.scp \
+          ${data}/$dset/utt2spk ${data}/$dset/raw.list
+    fi
+  done
+  # Convert all musan data to LMDB
+  python tools/make_lmdb.py ${data}/musan/wav.scp ${data}/musan/lmdb
+  # Convert all rirs data to LMDB
+  python tools/make_lmdb.py ${data}/rirs/wav.scp ${data}/rirs/lmdb
+fi
+```
+
+The voxceleb dataset contains millions of wav files. Frequently opening the large scale small files will cause the IO
+bottleneck. By default, the wav files from voxceleb dataset will be restored to some large binary shard files and the
+shard files' paths ared store in `$data/$dset/shard.list` file. In this script, the wav file number in each shard file
+is set to `1000`.
+
+Besides, the MUSAN and RIR_NOISES dataset are stored in LMDB format for fastly random-access in the training process.
+
+### Stage 3: Neural Network training
+
+```
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "Start training ..."
+  num_gpus=$(echo $gpus | awk -F ',' '{print NF}')
+  torchrun --standalone --nnodes=1 --nproc_per_node=$num_gpus \
+    wespeaker/bin/train.py --config $config \
+      --exp_dir ${exp_dir} \
+      --gpus $gpus \
+      --num_avg ${num_avg} \
+      --data_type "${data_type}" \
+      --train_data ${data}/vox2_dev/${data_type}.list \
+      --train_label ${data}/vox2_dev/utt2spk \
+      --reverb_data ${data}/rirs/lmdb \
+      --noise_data ${data}/musan/lmdb \
+      ${checkpoint:+--checkpoint $checkpoint}
+fi
+```
+
+The NN model is trained in this step.
+
+* Multi-GPU mode
+
+Here, the `torchrun` command is used to start `num_gpus` processes for pytorch DDP training. Set the gpus ids
+using `gpus` local variable. For example, `gpus="[0,1]"`, two gpus will be used and the used gpu idx is 0 and 1.
+
+* Model Initialization
+
+By default, the model is randomly initialized. You can also use some pre-trained model's weight to initialize the model
+by specify the `model_init` param in the config file.
+
+* Resume training
+
+If your experiment is terminated after running several epochs for some reasons (e.g. the GPU is accidentally used by
+other people and is out-of-memory ), you could continue the training from a checkpoint model. Just find out the finished
+epoch in `exp/your_exp/`, set `checkpoint=exp/your_exp/$n.pt` and run the `run.sh --stage 3`. Then the training will
+continue from the $n+1.pt
+
+* Config
+
+The config of neural network structure, optimization parameter, loss parameters, and dataset can be set in a YAML format
+file.
+
+Besides, under `conf/`, we have provide the configuration for different models, like ecapa, resnet, et al.
+
+### Stage 4: Speaker Embedding Extraction
+
+```
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  echo "Do model average ..."
+  avg_model=$exp_dir/models/avg_model.pt
+  python wespeaker/bin/average_model.py \
+    --dst_model $avg_model \
+    --src_path $exp_dir/models \
+    --num ${num_avg}
+
+  model_path=$avg_model
+  if [[ $config == *repvgg*.yaml ]]; then
+    echo "convert repvgg model ..."
+    python wespeaker/models/convert_repvgg.py \
+      --config $exp_dir/config.yaml \
+      --load $avg_model \
+      --save $exp_dir/models/convert_model.pt
+    model_path=$exp_dir/models/convert_model.pt
+  fi
+
+  echo "Extract embeddings ..."
+  local/extract_vox.sh \
+    --exp_dir $exp_dir --model_path $model_path \
+    --nj 4 --gpus $gpus --data_type $data_type --data ${data}
+fi
+```
+
+The embeddings for the train and evaluation set are extracted in this stage.
+
+* Average Model
+
+Average the model's weights from last `num_avg` checkpoints. This is a kind of model ensamble strategy to improve the
+system performance.
+
+* RepVGG Model Convert
+
+Because the RepVGG model have different forward paradigms for training and evaluation, here the model weight is
+converted to evaluation format.
+
+* Extract Embedding
+
+The extracted embeddings are stored in `exp/your_exp/embeddings` in kaldi scp,ark format. If there is someting wrong
+happened in this stage, you can check the log files under `exp/your_exp/embeddings/log` directory.
+
+### Stage 5: Scoring the Evaluation Set
+
+```
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+  echo "Score ..."
+  local/score.sh \
+    --stage 1 --stop-stage 2 \
+    --data ${data} \
+    --exp_dir $exp_dir \
+    --trials "$trials"
+fi
+```
+
+All the trails listed in local variable `trials` is scored in this stage. **Cosine similarity** is used to calculate the
+score for each trial pair. At the end of this stage, the Equal Error rate (EER), minDCF evaluation results are stored in
+the `exp/your_exp/scores/vox1_cos_result` file. Besides, the detailed score for each trial with trial_name `trial_xx`
+can be found in `exp/your_exp/scores/trial_xx.score` file.
+
+### Stage 6: Scoring the Evaluation Set
+
+```
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  echo "Score norm ..."
+  local/score_norm.sh \
+    --stage 1 --stop-stage 3 \
+    --score_norm_method $score_norm_method \
+    --cohort_set vox2_dev \
+    --top_n $top_n \
+    --data ${data} \
+    --exp_dir $exp_dir \
+    --trials "$trials"
+fi
+```
+
+Based on the score results in the last stage,
+the [adaptaive score normalization](https://www.isca-speech.org/archive/pdfs/interspeech_2017/matejka17_interspeech.pdf)
+is done to further improve the results. The final evaluation results are stored
+in `exp/your_exp/scores/vox1_${}${top_n}_result` file.
+
+`--score_norm_method`: asnorm or snorm, detailed algorithm can be found in
+this [paper](https://www.isca-speech.org/archive/pdfs/interspeech_2017/matejka17_interspeech.pdf).
+`--top_n`: the negative cohort size to calculate the adaptive statistics
+
+### Stage 7(Optional): Export the trained model
+
+```
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  # Export the best model you want
+  python wenet/bin/export_jit.py \
+    --config $dir/train.yaml \
+    --checkpoint $dir/avg_${average_num}.pt \
+    --output_file $dir/final.zip
+fi
+```
+
+`wenet/bin/export_jit.py` will export the trained model using Libtorch. The exported model files can be easily used for
+C++ inference in our runtime.
+
+### Stage 8(Optional): Large Margin Finetuning
+
+```
+if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
+  echo "Large margin fine-tuning ..."
+  lm_exp_dir=${exp_dir}-LM
+  mkdir -p ${lm_exp_dir}/models
+  # Use the pre-trained average model to initialize the LM training
+  cp ${exp_dir}/models/avg_model.pt ${lm_exp_dir}/models/model_0.pt
+  bash run.sh --stage 3 --stop_stage 7 \
+      --data ${data} \
+      --data_type ${data_type} \
+      --config ${lm_config} \
+      --exp_dir ${lm_exp_dir} \
+      --gpus $gpus \
+      --num_avg 1 \
+      --checkpoint ${lm_exp_dir}/models/model_0.pt \
+      --trials "$trials" \
+      --score_norm_method ${score_norm_method} \
+      --top_n ${top_n}
+fi
+```
+
+This [paper](https://arxiv.org/pdf/2010.11255.pdf) has shown that finetuning the model for another few epoches
+by increasing the training segment duration and enlarging the margin in the loss function at the same time can further
+improve the performance for Voxceleb data. Users can run this stage for the better results.
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/vox_ssl.md b/MOSS_Speaker-RoPE/wespeaker/docs/vox_ssl.md
new file mode 100644
index 0000000000000000000000000000000000000000..1d0541970d64cb7787b3a0dcb386e49403ff0037
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/vox_ssl.md
@@ -0,0 +1,325 @@
+## SV Tutorial on VoxCeleb v3 (Self-Supervised)
+
+If you meet any problems when going through this tutorial, please feel free to ask in
+github [issues](https://github.com/wenet-e2e/wespeaker/issues). Thanks for any kind of feedback.
+
+### First Experiment
+
+We provide three self-supervised recipes on voxceleb data. They are currently the three most commonly used frameworks
+for self-supervised speaker verification. If you want to learn more, you can refer to the `README.md` in the
+corresponding directories.
+
+* SimCLR: `examples/voxceleb/v3/simclr/run.sh`
+* MoCo: `examples/voxceleb/v3/moco/run.sh`
+* DINO: `examples/voxceleb/v3/dino/run.sh`
+
+Because the steps of these three algorithms are basically the same, the following tutorial will take **DINO** as an
+example. The recipe is simple and we suggest you run each stage one by one manually and check the result to understand
+the whole processs.
+
+```
+cd examples/voxceleb/v3/dino
+bash run.sh --stage 1 --stop_stage 1
+bash run.sh --stage 2 --stop_stage 2
+bash run.sh --stage 3 --stop_stage 3
+bash run.sh --stage 4 --stop_stage 4
+bash run.sh --stage 5 --stop_stage 5
+bash run.sh --stage 6 --stop_stage 6
+```
+
+### Stage 1: Download Data
+
+```
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "Prepare datasets ..."
+  ./local/prepare_data.sh --stage 2 --stop_stage 4 --data ${data}
+fi
+```
+
+This step is exactly the same as the recipe for supervised training on voxceleb `examples/voxceleb/v2`. **If you have
+done it before, you can skip this step.**
+
+This stage prepares the **voxceleb1**, **voxceleb2**, **MUSAN** and **RIRS_NOISES** dataset. MUSAN is a noise dataset
+and RIRS_NOISES is a reverberation dataset, which are used for data augmentation. It should be noted that for
+self-supervised speaker verification, data augmentation is crucial for the training process. We strongly recommend
+incorporating MUSAN and RIRS_NOISES data augmentation here.
+
+It should be noted that the `./local/prepare_data.sh` script starts from the stage 2. It is because the data downloading
+process in stage 1 will take a long time. Thus we recommand you to download all archives above in your own way first and
+put it under `data/download_data` and then run the above script.
+
+When finishing this stage, you will get the following meta files:
+
+* **wav.scp** files for all the dataset:
+    * `data/musan/wav.scp`
+    * `data/rirs/wav.scp`
+    * `data/vox1/wav.scp`
+    * `data/vox2_dev/wav.scp`
+* **utt2spk** and **spk2utt** files for voxceleb1 and voxceleb2_dev
+    * `data/vox1/utt2spk`
+    * `data/vox1/spk2utt`
+    * `data/vox2_dev/utt2spk`
+    * `data/vox2_dev/spk2utt`
+* **trials**
+    * `data/vox1/trials/vox1_O_cleaned.kaldi`
+    * `data/vox1/trials/vox1_E_cleaned.kaldi`
+    * `data/vox1/trials/vox1_H_cleaned.kaldi`
+
+**wav.scp** each line records two blank-separated columns : `wav_id` and `wav_path`
+
+```
+id10001/1zcIwhmdeo4/00001.wav /exported/data/voxceleb1_wav_v2/id10001/1zcIwhmdeo4/00001.wav
+id10001/1zcIwhmdeo4/00002.wav /exported/data/voxceleb1_wav_v2/id10001/1zcIwhmdeo4/00002.wav
+...
+```
+
+**utt2spk** each line records two blank-separated columns : `wav_id` and `spk_id`
+
+```
+id10001/1zcIwhmdeo4/00001.wav id10001
+id10001/1zcIwhmdeo4/00002.wav id10001
+...
+```
+
+**spk2utt** each line records many blank-separated columns : `spk_id` and many `wav_id`s belong to this `spk_id`
+
+```
+id10001 id10001/1zcIwhmdeo4/00001.wav id10001/1zcIwhmdeo4/00002.wav id10001/1zcIwhmdeo4/00003.wav ...
+id10002 id10002/0_laIeN-Q44/00001.wav id10002/6WO410QOeuo/00001.wav ...
+...
+```
+
+**trials** each line records three blank-separated columns : `enroll_wav_id`, `test_wav_id` and `label`
+
+```
+id10001/Y8hIVOBuels/00001.wav id10001/1zcIwhmdeo4/00001.wav target
+id10001/Y8hIVOBuels/00001.wav id10943/vNCVj7yLWPU/00005.wav nontarget
+id10001/Y8hIVOBuels/00001.wav id10001/7w0IBEWc9Qw/00004.wav target
+id10001/Y8hIVOBuels/00001.wav id10999/G5R2-Hl7YX8/00008.wav nontarget
+...
+```
+
+In this step, we generated **utt2spk** and **spk2utt**, but we will not use any speaker labels during the training
+process.
+
+### Stage 2: Reformat the Data
+
+```
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "Covert train and test data to ${data_type}..."
+  for dset in vox2_dev vox1; do
+    if [ $data_type == "shard" ]; then
+      python tools/make_shard_list.py --num_utts_per_shard 1000 \
+          --num_threads 16 \
+          --prefix shards \
+          --shuffle \
+          ${data}/$dset/wav.scp ${data}/$dset/utt2spk \
+          ${data}/$dset/shards ${data}/$dset/shard.list
+    else
+      python tools/make_raw_list.py ${data}/$dset/wav.scp \
+          ${data}/$dset/utt2spk ${data}/$dset/raw.list
+    fi
+  done
+  # Convert all musan data to LMDB
+  python tools/make_lmdb.py ${data}/musan/wav.scp ${data}/musan/lmdb
+  # Convert all rirs data to LMDB
+  python tools/make_lmdb.py ${data}/rirs/wav.scp ${data}/rirs/lmdb
+fi
+```
+
+This step is exactly the same as the recipe for supervised training on voxceleb `examples/voxceleb/v2`. **If you have
+done it before, you can skip this step.**
+
+The voxceleb dataset contains millions of wav files. Frequently opening the large scale small files will cause the IO
+bottleneck. By default, the wav files from voxceleb dataset will be restored to some large binary shard files and the
+shard files' paths ared store in `$data/$dset/shard.list` file. In this script, the wav file number in each shard file
+is set to `1000`.
+
+Besides, the MUSAN and RIR_NOISES dataset are stored in LMDB format for fastly random-access in the training process.
+
+### Stage 3: Neural Network training
+
+```
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "Start training ..."
+  num_gpus=$(echo $gpus | awk -F ',' '{print NF}')
+  torchrun --standalone --nnodes=1 --nproc_per_node=$num_gpus \
+    wespeaker/ssl/bin/train_dino.py --config $config \
+      --exp_dir ${exp_dir} \
+      --gpus $gpus \
+      --num_avg ${num_avg} \
+      --data_type "${data_type}" \
+      --train_data ${data}/vox2_dev/${data_type}.list \
+      --wav_scp ${data}/vox2_dev/wav.scp \
+      --reverb_data ${data}/rirs/lmdb \
+      --noise_data ${data}/musan/lmdb \
+      ${checkpoint:+--checkpoint $checkpoint}
+fi
+```
+
+The NN model is trained in this step.
+
+* Multi-GPU mode
+
+Here, the `torchrun` command is used to start `num_gpus` processes for pytorch DDP training. Set the gpus ids
+using `gpus` local variable. For example, `gpus="[0,1]"`, two gpus will be used and the used gpu idx is 0 and 1.
+
+* Model Initialization
+
+By default, the model is randomly initialized. You can also use some pre-trained model's weight to initialize the model
+by specify the `model_init` param in the config file.
+
+* Resume training
+
+If your experiment is terminated after running several epochs for some reasons (e.g. the GPU is accidentally used by
+other people and is out-of-memory ), you could continue the training from a checkpoint model. Just find out the finished
+epoch in `exp/your_exp/`, set `checkpoint=exp/your_exp/$n.pt` and run the `run.sh --stage 3`. Then the training will
+continue from the $n+1.pt
+
+* Config
+
+The config of neural network structure, optimization parameter, loss parameters, and dataset can be set in a YAML format
+file.
+
+Besides, under `conf/`, we have provide the configuration for different models, like ecapa `conf/ecapa.yaml` and
+resnet `conf/resnet34.yaml`.
+
+* Self-supervised Training Related
+
+It's should be noted that for MoCo and SimCLR, the python scripts should be `wespeaker/ssl/bin/train_contrastive.py`.
+
+The biggest difference compared to supervised training recipe `examples/voxceleb/v2` is the way data is organized.
+Because self supervised training cannot use real labels, it can only construct sample pairs for contrastive learning
+through assumptions. (a) The segments cropped from the same utterance belong to the same speaker (b) The segments
+cropped from different utterances belong to different speakers.
+
+For self-suerpervised training recipes, **dataloader** is defined in `wespeaker/ssl/dataset/dataset.py`. Next, I will
+briefly introduce the process of dataset.
+
+Firstly, we define different data reading methods based on different data storage formats. And perform global and local
+shuffling.
+
+```
+dataset = DataList(lists, shuffle=shuffle)
+if data_type == 'shard':
+    dataset = Processor(dataset, processor.url_opener)
+    dataset = Processor(dataset, processor.tar_file_and_group)
+elif data_type == 'raw':
+    dataset = Processor(dataset, processor.parse_raw)
+else:
+    dataset = Processor(dataset, processor.parse_feat)
+# Local shuffle
+if shuffle:
+    dataset = Processor(dataset, processor.shuffle,
+                        **configs['shuffle_args'])
+```
+
+Then, we defined different sample pair composition methods for different training methods. For SimCLR and MoCo, we take
+2 segments from each sentence randomly. For DINO, we will crop 2 short and 4 long segments to form a positive pair.
+
+```
+# random chunk
+frame_shift = configs['fbank_args'].get('frame_shift', 10)
+frame_length = configs['fbank_args'].get('frame_length', 25)
+chunk_info_args = configs['chunk_info_args']
+for key in chunk_info_args:
+    if 'chunk_len' in key:
+        chunk_info_args[key] = (
+            (chunk_info_args[key] - 1) * frame_shift +
+            frame_length) * resample_rate // 1000
+chunk_info_args['data_type'] = data_type
+dataset = Processor(dataset, ssl_processor.random_chunk_for_dino,
+                    **chunk_info_args)
+```
+
+Finally, it is a very important data augmentation step. We will randomly apply different data augmentation strategies to
+each segment here.
+
+```
+# add reverb & noise
+aug_prob = configs.get('aug_prob', 0.6)
+if (reverb_lmdb_file and noise_lmdb_file) and (aug_prob > 0.0):
+    reverb_data = LmdbData(reverb_lmdb_file)
+    noise_data = LmdbData(noise_lmdb_file)
+    dataset = Processor(dataset, ssl_processor.add_reverb_noise,
+                        reverb_data, noise_data, resample_rate,
+                        aug_prob)
+```
+
+Wespeaker notably facilitates effortless configuration for organizing diverse processors into a pipeline, ensuring both
+efficiency and ease of extension. And the SSL related processors are defined in `wespeaker/ssl/dataset/processor.py`
+
+In addition, in order to be more compatible with the existing framework of WeSpeaker, we have added wrappers to the
+training models of SimCLR, MoCo, and DINO, which are defined in `wespeaker/ssl/models`. It includes **additional modules
+** required for SSL training, the definition of **loss functions** and so on.
+
+### Stage 4: Speaker Embedding Extraction
+
+```
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  echo "Do model average ..."
+  avg_model=$exp_dir/models/avg_model.pt
+  python wespeaker/ssl/bin/average_dino_model.py \
+    --dst_model $avg_model \
+    --src_path $exp_dir/models \
+    --num ${num_avg}
+
+  echo "Extract embeddings ..."
+  local/extract_vox.sh \
+    --exp_dir $exp_dir --model_path $avg_model \
+    --nj 4 --gpus $gpus --data_type $data_type --data ${data}
+fi
+```
+
+The embeddings for the train and evaluation set are extracted in this stage.
+
+* Average Model
+
+Average the model's weights from last `num_avg` checkpoints. This is a kind of model ensamble strategy to improve the
+system performance.
+
+It's should be noted that for MoCo and SimCLR, the python scripts should
+be `wespeaker/ssl/bin/average_contrastive_model.py`. Because self-supervised training frameworks generally require the
+introduction of additional modules (such as student model, projection head et al.) to assist in training, it is
+necessary to remove these additional modules in this step to facilitate subsequent feature extraction.
+
+* Extract Embedding
+
+The extracted embeddings are stored in `exp/your_exp/embeddings` in kaldi scp,ark format. If there is something wrong
+happened in this stage, you can check the log files under `exp/your_exp/embeddings/log` directory.
+
+### Stage 5: Scoring the Evaluation Set
+
+```
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+  echo "Score ..."
+  local/score.sh \
+    --stage 1 --stop-stage 2 \
+    --data ${data} \
+    --exp_dir $exp_dir \
+    --trials "$trials"
+fi
+```
+
+All the trails listed in local variable `trials` is scored in this stage. **Cosine similarity** is used to calculate the
+score for each trial pair. At the end of this stage, the Equal Error rate (EER), minDCF evaluation results are stored in
+the `exp/your_exp/scores/vox1_cos_result` file. Besides, the detailed score for each trial with trial_name `trial_xx`
+can be found in `exp/your_exp/scores/trial_xx.score` file.
+
+Unlike supervised training recipe, we will not perform asnorm here because theoretically we cannot use any voxceleb
+labels for score normalization.
+
+### Stage 6(Optional): Export the trained model
+
+```
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  echo "Export the best model ..."
+  python wespeaker/bin/export_jit.py \
+    --config $exp_dir/config.yaml \
+    --checkpoint $exp_dir/models/avg_model.pt \
+    --output_file $exp_dir/models/final.zip
+fi
+```
+
+`wenet/bin/export_jit.py` will export the trained model using Libtorch. The exported model files can be easily used for
+C++ inference in our runtime.
diff --git a/MOSS_Speaker-RoPE/wespeaker/docs/voxconverse_diar.md b/MOSS_Speaker-RoPE/wespeaker/docs/voxconverse_diar.md
new file mode 100644
index 0000000000000000000000000000000000000000..e60819faece8760c079988592ab7d719a8a53d23
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/docs/voxconverse_diar.md
@@ -0,0 +1,314 @@
+## Diarization Tutorial on VoxConverse v2
+
+If you meet any problems when going through this tutorial, please feel free to ask in github [issues](https://github.com/wenet-e2e/wespeaker/issues). Thanks for any kind of feedback.
+
+
+### First Experiment
+
+Speaker diarization is a typical downstream task of applying the well-learnt speaker embedding.
+Here we introduce our diarization recipe `examples/voxconverse/v2/run.sh` on the Voxconverse 2020 dataset.
+
+Note that we provide two recipes: **v1** and **v2**. Their only difference is that in **v2**, we split the Fbank extraction, embedding extraction and clustering modules to different stages.
+We recommend newcomers to follow the **v2** recipe and run it stage by stage and check the result to better understand the whole process.
+
+```
+cd examples/voxconverse/v2/
+bash run.sh --stage 1 --stop_stage 1
+bash run.sh --stage 2 --stop_stage 2
+bash run.sh --stage 3 --stop_stage 3
+bash run.sh --stage 4 --stop_stage 4
+bash run.sh --stage 5 --stop_stage 5
+bash run.sh --stage 6 --stop_stage 6
+bash run.sh --stage 7 --stop_stage 7
+bash run.sh --stage 8 --stop_stage 8
+```
+
+
+### Stage 1: Download Prerequisites
+
+```
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    mkdir -p external_tools
+
+    # [1] Download evaluation toolkit
+    wget -c https://github.com/usnistgov/SCTK/archive/refs/tags/v2.4.12.zip -O external_tools/SCTK-v2.4.12.zip
+    unzip -o external_tools/SCTK-v2.4.12.zip -d external_tools
+
+    # [2] Download voice activity detection model pretrained by Silero Team
+    wget -c https://github.com/snakers4/silero-vad/archive/refs/tags/v3.1.zip -O external_tools/silero-vad-v3.1.zip
+    unzip -o external_tools/silero-vad-v3.1.zip -d external_tools
+
+    # [3] Download ResNet34 speaker model pretrained by WeSpeaker Team
+    mkdir -p pretrained_models
+
+    wget -c https://wespeaker-1256283475.cos.ap-shanghai.myqcloud.com/models/voxceleb/voxceleb_resnet34_LM.onnx -O pretrained_models/voxceleb_resnet34_LM.onnx
+fi
+```
+
+Download three Prerequisites:
+* the evaluation toolkit **SCTK**: Compute the DER metric
+* the open-source VAD model pre-trained by [silero-vad](https://github.com/snakers4/silero-vad): Remove the silence in audio
+* the pre-trained ResNet34 model: Extract the speaker embeddings
+
+When finishing this stage, you will get two new dirs:
+- **external_tools**
+    - SCTK-v2.4.12.zip
+    - SCTK-v2.4.12
+    - silero-vad-v3.1.zip
+    - silero-vad-v3.1
+- **pretrained_models**
+    - voxceleb_resnet34_LM.onnx
+
+
+### Stage 2: Download and Prepare Data
+
+```
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    mkdir -p data
+
+    # Download annotations for dev and test sets (version 0.0.3)
+    wget -c https://github.com/joonson/voxconverse/archive/refs/heads/master.zip -O data/voxconverse_master.zip
+    unzip -o data/voxconverse_master.zip -d data
+
+    # Download annotations from VoxSRC-23 validation toolkit (looks like version 0.0.2)
+    # cd data && git clone https://github.com/JaesungHuh/VoxSRC2023.git --recursive && cd -
+
+    # Download dev audios
+    mkdir -p data/dev
+
+    wget --no-check-certificate -c https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_dev_wav.zip -O data/voxconverse_dev_wav.zip
+    unzip -o data/voxconverse_dev_wav.zip -d data/dev
+
+    # Create wav.scp for dev audios
+    ls `pwd`/data/dev/audio/*.wav | awk -F/ '{print substr($NF, 1, length($NF)-4), $0}' > data/dev/wav.scp
+
+    # Test audios
+    mkdir -p data/test
+
+    wget  --no-check-certificate -c https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_test_wav.zip -O data/voxconverse_test_wav.zip
+    unzip -o data/voxconverse_test_wav.zip -d data/test
+
+    # Create wav.scp for test audios
+    ls `pwd`/data/test/voxconverse_test_wav/*.wav | awk -F/ '{print substr($NF, 1, length($NF)-4), $0}' > data/test/wav.scp
+fi
+```
+
+Download the Voxconverse 2020 dev and test sets as well as their annotations.
+Here we use the latest version 0.0.3 in default (recommended).
+You can also try the version 0.0.2 (seem to be used in the [VoxSRC-23 baseline repo](https://github.com/JaesungHuh/VoxSRC2023.git)).
+
+When finishing this stage, you will get the new **data** dir:
+- **data**
+    - voxconverse_master.zip
+    - voxconverse_dev_wav.zip
+    - voxconverse_test_wav.zip
+    - voxconverse_master
+        - dev: ground-truth rttms
+        - test: ground-truth rttms
+    - dev
+        - audio: wav files
+        - wav.scp
+    - test
+        - voxconverse_test_wav: wav files
+        - wav.scp
+
+**wav.scp**: each line records two blank-separated columns : `wav_id` and `wav_path`
+```
+abjxc /path/to/wespeaker/examples/voxconverse/v2/data/dev/audio/abjxc.wav
+afjiv /path/to/wespeaker/examples/voxconverse/v2/data/dev/audio/afjiv.wav
+...
+```
+
+
+### Stage 3: Apply SAD (i.e., VAD)
+
+```
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+    # Set VAD min duration
+    min_duration=0.255
+
+    if [[ "x${sad_type}" == "xoracle" ]]; then
+        # Oracle SAD: handling overlapping or too short regions in ground truth RTTM
+        while read -r utt wav_path; do
+            python3 wespeaker/diar/make_oracle_sad.py \
+                    --rttm data/voxconverse-master/${partition}/${utt}.rttm \
+                    --min-duration $min_duration
+        done < data/${partition}/wav.scp > data/${partition}/oracle_sad
+    fi
+
+    if [[ "x${sad_type}" == "xsystem" ]]; then
+       # System SAD: applying 'silero' VAD
+       python3 wespeaker/diar/make_system_sad.py \
+               --repo-path external_tools/silero-vad-3.1 \
+               --scp data/${partition}/wav.scp \
+               --min-duration $min_duration > data/${partition}/system_sad
+    fi
+fi
+```
+
+`sad_type` could be oracle or system:
+* oracle: get vad infos from the ground truth RTTMs, saved in `data/${partition}/oracle_sad`
+* system: compute vad results using the [silero-vad](https://github.com/snakers4/silero-vad), saved in `data/${partition}/system_sad`
+
+where `partition` is dev or test.
+
+Note that too short VAD segments with less than `min_duration` seconds are ignored and simply regarded as silence.
+
+
+### Stage 4: Extract Fbank Features
+
+```
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+
+    [ -d "exp/${sad_type}_sad_fbank" ] && rm -r exp/${sad_type}_sad_fbank
+
+    echo "Make Fbank features and store it under exp/${sad_type}_sad_fbank"
+    echo "..."
+    bash local/make_fbank.sh \
+            --scp data/${partition}/wav.scp \
+            --segments data/${partition}/${sad_type}_sad \
+            --store_dir exp/${partition}_${sad_type}_sad_fbank \
+            --subseg_cmn ${subseg_cmn} \
+            --nj 24
+fi
+```
+
+`subseg_cmn` suggests applying Cepstral Mean Normalization (CMN) to Fbanks:
+* on the sliding-window sub-segment (`subseg_cmn=true`) or
+* on the whole vad segment (`subseg_cmn=false`)
+
+You can specify `nj` jobs according to your cpu cores num.
+The final Fbank features are saved under dir `exp/${partition}_${sad_type}_sad_fbank`.
+
+
+### Stage 5: Extract Sliding-window Speaker Embeddings
+
+```
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+
+    [ -d "exp/${sad_type}_sad_embedding" ] && rm -r exp/${sad_type}_sad_embedding
+
+    echo "Extract embeddings and store it under exp/${sad_type}_sad_embedding"
+    echo "..."
+    bash local/extract_emb.sh \
+            --scp exp/${partition}_${sad_type}_sad_fbank/fbank.scp \
+            --pretrained_model pretrained_models/voxceleb_resnet34_LM.onnx \
+            --device cuda \
+            --store_dir exp/${partition}_${sad_type}_sad_embedding \
+            --batch_size 96 \
+            --frame_shift 10 \
+            --window_secs 1.5 \
+            --period_secs 0.75 \
+            --subseg_cmn ${subseg_cmn} \
+            --nj 1
+fi
+```
+
+Extract speaker embeddings from the Fbank features in a sliding-window fashion: `step=0.75s, window=1.5s`,
+which means extracting embedding from each `1.5s` speech window after every `0.75s`.
+Thus the contiguous windows overlap by `1.5-0.75=0.75s` in duration.
+
+You can also specify `nj` jobs and decide to use the `gpu` or `cpu` devices.
+The extracted embeddings are saved under dir `exp/${partition}_${sad_type}_sad_embedding`.
+
+
+### Stage 6: Apply Spectral Clustering
+
+```
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+
+    [ -f "exp/spectral_cluster/${partition}_${sad_type}_sad_labels" ] && rm exp/spectral_cluster/${partition}_${sad_type}_sad_labels
+
+    echo "Doing spectral clustering and store the result in exp/spectral_cluster/${partition}_${sad_type}_sad_labels"
+    echo "..."
+    python3 wespeaker/diar/spectral_clusterer.py \
+            --scp exp/${partition}_${sad_type}_sad_embedding/emb.scp \
+            --output exp/spectral_cluster/${partition}_${sad_type}_sad_labels
+fi
+```
+
+Apply spectral clustering using the extracted sliding-window speaker embeddings,
+and store the results in `exp/spectral_cluster/${partition}_${sad_type}_sad_labels`,
+where each line records two blank-separated columns : `subseg_id` and `spk_id`
+```
+abjxc-00000400-00007040-00000000-00000150 0
+abjxc-00000400-00007040-00000075-00000225 0
+abjxc-00000400-00007040-00000150-00000300 0
+abjxc-00000400-00007040-00000225-00000375 0
+...
+```
+
+
+### Stage 7: Reformat Clustering Labels into RTTMs
+
+```
+if [ ${stage} -le 7 ] && [ ${stop_stage} -ge 7 ]; then
+    python3 wespeaker/diar/make_rttm.py \
+            --labels exp/spectral_cluster/${partition}_${sad_type}_sad_labels \
+            --channel 1 > exp/spectral_cluster/${partition}_${sad_type}_sad_rttm
+fi
+```
+
+Convert the clustering labels into the Rich Transcription Time Marked (RTTM) format, saved in `exp/spectral_cluster/${partition}_${sad_type}_sad_rttm`.
+
+RTTM files are space-delimited text files containing one turn per line, each line containing ten fields:
+
+* `Type` -- segment type; should always by `SPEAKER`
+* `File ID` -- file name; basename of the recording minus extension (e.g., `abjxc`)
+* `Channel ID` -- channel (1-indexed) that turn is on; should always be `1`
+* `Turn Onset` -- onset of turn in seconds from beginning of recording
+* `Turn Duration` -- duration of turn in seconds
+* `Orthography Field` -- should always by `<NA>`
+* `Speaker Type` -- should always be `<NA>`
+* `Speaker Name` -- name of speaker of turn; should be unique within scope of each file
+* `Confidence Score` -- system confidence (probability) that information is correct; should always be `<NA>`
+* `Signal Lookahead Time` -- should always be `<NA>`
+
+For instance,
+
+```
+SPEAKER abjxc 1 0.400 6.640 <NA> <NA> 0 <NA> <NA>
+SPEAKER abjxc 1 8.680 55.960 <NA> <NA> 0 <NA> <NA>
+```
+
+
+### Stage 8: Evaluate the Result (DER)
+
+```
+if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
+    ref_dir=data/voxconverse-master/
+    #ref_dir=data/VoxSRC2023/voxconverse/
+    echo -e "Get the DER results\n..."
+    perl external_tools/SCTK-2.4.12/src/md-eval/md-eval.pl \
+         -c 0.25 \
+         -r <(cat ${ref_dir}/${partition}/*.rttm) \
+         -s exp/spectral_cluster/${partition}_${sad_type}_sad_rttm 2>&1 | tee exp/spectral_cluster/${partition}_${sad_type}_sad_res
+
+    if [ ${get_each_file_res} -eq 1 ];then
+        single_file_res_dir=exp/spectral_cluster/${partition}_${sad_type}_single_file_res
+        mkdir -p $single_file_res_dir
+        echo -e "\nGet the DER results for each file and the results will be stored underd ${single_file_res_dir}\n..."
+
+        awk '{print $2}' exp/spectral_cluster/${partition}_${sad_type}_sad_rttm | sort -u  | while read file_name; do
+            perl external_tools/SCTK-2.4.12/src/md-eval/md-eval.pl \
+                 -c 0.25 \
+                 -r <(cat ${ref_dir}/${partition}/${file_name}.rttm) \
+                 -s <(grep "${file_name}" exp/spectral_cluster/${partition}_${sad_type}_sad_rttm) > ${single_file_res_dir}/${partition}_${file_name}_res
+        done
+        echo "Done!"
+    fi
+fi
+```
+
+Use the **SCTK** toolkit to compute the Diarization Error Rate (DER) metric, which is the sum of
+
+* speaker error -- percentage of scored time for which the wrong speaker id is assigned within a speech region
+* false alarm speech -- percentage of scored time for which a nonspeech region is incorrectly marked as containing speech
+* missed speech -- percentage of scored time for which a speech region is incorrectly marked as not containing speech
+
+For more details about DER, consult Section 6.1 of the [NIST RT-09 evaluation plan](https://web.archive.org/web/20100606092041if_/http://www.itl.nist.gov/iad/mig/tests/rt/2009/docs/rt09-meeting-eval-plan-v2.pdf).
+
+The overall DER result would be saved in `exp/spectral_cluster/${partition}_${sad_type}_sad_res`.
+Optionally, set `get_each_file_res` as `1` if you also want to get the DER result for each single file, which will be saved under dir `exp/spectral_cluster/${partition}_${sad_type}_single_file_res`.
+
+
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/README.md b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..32177eeedfa22fdf086ca7ee024dea1e1859bc1d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/README.md
@@ -0,0 +1,35 @@
+## Results
+
+* Setup: fbank80, num_frms200, epoch150, ArcMargin, aug_prob0.6, speed_perturb (no spec_aug)
+* Scoring: cosine (sub mean of vox2_dev), AS-Norm, [QMF](https://arxiv.org/pdf/2010.11255)
+* Test_trial: CNC-Eval-Avg.lst
+
+* 🔥 UPDATE 2022.07.12: We update this recipe according to the setups in the winning system of CNSRC 2022, and get obvious performance improvement compared with the old recipe. Check the [commit1](https://github.com/wenet-e2e/wespeaker/pull/63/commits/b08804987b3bbb26f4963cedf634058474c743dd), [commit2](https://github.com/wenet-e2e/wespeaker/pull/66/commits/6f6af29197f0aa0a5d1b1993b7feb2f41b97891f) for details.
+    * LR scheduler warmup from 0
+    * Remove one embedding layer
+    * Add large margin fine-tuning strategy (LM)
+
+| Model                             | Params    | FLOPs   | LM  | AS-Norm   | QMF | EER (%)   | minDCF (p=0.01)  |
+| :------------------------------   | :-------: | :-----: | :-: | :-------: | :-: | :-------: | :--------------: |
+| ResNet34-TSTP-emb256 (OLD)        | 6.70M     | 4.55 G  | ×   | ×         | ×   | 8.426     | 0.487            |
+| ResNet34-TSTP-emb256              | 6.63M     | 4.55 G  | ×   | ×         | ×   | 7.134     | 0.408            |
+|                                   |           |         | ×   | √         | ×   | 6.747     | 0.367            |
+|                                   |           |         | ×   | √         | √   | 6.336     | 0.374            |
+|                                   |           |         | √   | ×         | ×   | 6.652     | 0.393            |
+|                                   |           |         | √   | √         | ×   | 6.492     | 0.354            |
+|                                   |           |         | √   | √         | √   | 6.119     | 0.361            |
+| ResNet221-TSTP-emb256             | 23.86M    | 21.29 G | ×   | ×         | ×   | 5.965     | 0.362            |
+|                                   |           |         | ×   | √         | ×   | 5.708     | **0.326**        |
+|                                   |           |         | √   | ×         | ×   | 5.886     | 0.362            |
+|                                   |           |         | √   | √         | ×   | **5.655** | 0.330            |
+| ECAPA_TDNN_GLOB_c512-ASTP-emb192  | 6.19M     | 1.04 G  | ×   | ×         | ×   | 8.313     | 0.432            |
+|                                   |           |         | ×   | √         | ×   | 7.644     | 0.390            |
+|                                   |           |         | √   | ×         | ×   | 8.004     | 0.422            |
+|                                   |           |         | √   | √         | ×   | 7.417     | 0.379            |
+| ECAPA_TDNN_GLOB_c1024-ASTP-emb192 | 14.65M    | 2.65 G  | ×   | ×         | ×   | 7.879     | 0.420            |
+|                                   |           |         | ×   | √         | ×   | 7.412     | 0.379            |
+|                                   |           |         | √   | ×         | ×   | 7.986     | 0.417            |
+|                                   |           |         | √   | √         | ×   | 7.395     | 0.372            |
+| RepVGG_TINY_A0                    | 6.26M     | 4.65 G  | ×   | ×         | ×   | 6.883     | 0.399            |
+|                                   |           |         | ×   | √         | ×   | 6.550     | 0.355            |
+
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cfedf6832c330c766f609e5023da74e3993902cf
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn.yaml
@@ -0,0 +1,82 @@
+### train configuraton
+
+exp_dir: exp/ECAPA_TDNN_GLOB_c512-ASTP-emb192-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150
+gpus: "[0,1]"
+num_avg: 10
+enable_amp: False # whether enable automatic mixed precision training
+
+seed: 42
+num_epochs: 150
+save_epoch_interval: 5 # save model every 5 epochs
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 128
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 200
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ECAPA_TDNN_GLOB_c512 # ECAPA_TDNN_GLOB_c512, ECAPA_TDNN_GLOB_c1024
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 192
+  pooling_func: "ASTP"
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.0
+  final_margin: 0.2
+  increase_start_epoch: 20
+  fix_start_epoch: 40
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 0.1
+  final_lr: 0.00005
+  warm_up_epoch: 6
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn_lm.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn_lm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1db1556ce8e4d41fbbd37a390fc3efac6c9f7a65
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/ecapa_tdnn_lm.yaml
@@ -0,0 +1,88 @@
+### Large margin fine-tuning configuration
+#
+#   The large margin fine-tuning operation is often used in speaker
+#   verification challenge system to further improve the performance.
+#   In this fine-tuning stage, large margin and longer segment will
+#   be used.
+
+exp_dir: exp/ECAPA_TDNN_GLOB_c512-ASTP-emb192-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150-LM
+gpus: "[0,1]"
+num_avg: 1
+enable_amp: False # whether enable automatic mixed precision training
+do_lm: True
+
+seed: 42
+num_epochs: 5
+save_epoch_interval: 1 # save model per epoch
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 64
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 600
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ECAPA_TDNN_GLOB_c512 # ECAPA_TDNN_GLOB_c512, ECAPA_TDNN_GLOB_c1024
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 192
+  pooling_func: "ASTP"
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.5
+  final_margin: 0.5
+  increase_start_epoch: 1
+  fix_start_epoch: 1
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 1.0e-4
+  final_lr: 2.5e-5
+  warm_up_epoch: 1
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/repvgg.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/repvgg.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b47101e01ec1e94ef6f81564e3a95c66dbea1b7c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/repvgg.yaml
@@ -0,0 +1,84 @@
+### train configuraton
+
+exp_dir: exp/RepVGG-TSTP-emb256-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150
+gpus: "[0,1]"
+num_avg: 10
+enable_amp: False # whether enable automatic mixed precision training
+
+seed: 42
+num_epochs: 150
+save_epoch_interval: 5 # save model every 5 epochs
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 128
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 200
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: REPVGG_TINY_A0 # REPVGG_A0 REPVGG_A1 REPVGG_A2 REPVGG_RSBB_A0 REPVGG_RSBB_A2
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 256 # 512
+  pooling_func: "TSTP"
+  deploy: False
+  use_se: False
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.0
+  final_margin: 0.2
+  increase_start_epoch: 20
+  fix_start_epoch: 40
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 0.1
+  final_lr: 0.00005
+  warm_up_epoch: 6
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4a533aad51f60cd7f2927e9833caf837aa655307
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet.yaml
@@ -0,0 +1,83 @@
+### train configuraton
+
+exp_dir: exp/ResNet34-TSTP-emb256-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150
+gpus: "[0,1]"
+num_avg: 10
+enable_amp: False # whether enable automatic mixed precision training
+
+seed: 42
+num_epochs: 150
+save_epoch_interval: 5 # save model every 5 epochs
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 128
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 200
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ResNet34 # ResNet18, ResNet34, ResNet50, ResNet101, ResNet152
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 256
+  pooling_func: "TSTP"
+  two_emb_layer: False
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.0
+  final_margin: 0.2
+  increase_start_epoch: 20
+  fix_start_epoch: 40
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 0.1
+  final_lr: 0.00005
+  warm_up_epoch: 6
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet_lm.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet_lm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fd0a93c749b10f7d51f87fbb6f9896f5ce43afbd
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/conf/resnet_lm.yaml
@@ -0,0 +1,89 @@
+### Large margin fine-tuning configuration
+#
+#   The large margin fine-tuning operation is often used in speaker
+#   verification challenge system to further improve the performance.
+#   In this fine-tuning stage, large margin and longer segment will
+#   be used.
+
+exp_dir: exp/ResNet34-TSTP-emb256-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150-LM
+gpus: "[0,1]"
+num_avg: 1
+enable_amp: False # whether enable automatic mixed precision training
+do_lm: True
+
+seed: 42
+num_epochs: 5
+save_epoch_interval: 1 # save model per epoch
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 64
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 600
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ResNet34 # ResNet18, ResNet34, ResNet50, ResNet101, ResNet152
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 256
+  pooling_func: "TSTP"
+  two_emb_layer: False
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.5
+  final_margin: 0.5
+  increase_start_epoch: 1
+  fix_start_epoch: 1
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 1.0e-4
+  final_lr: 2.5e-5
+  warm_up_epoch: 1
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/choose_utts_to_combine.py b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/choose_utts_to_combine.py
new file mode 100644
index 0000000000000000000000000000000000000000..f718abb1f99c994a9e836d1ede503be7f15df61d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/choose_utts_to_combine.py
@@ -0,0 +1,433 @@
+#!/usr/bin/env python
+
+# Copyright 2016  Vijayaditya Peddinti
+#           2016  Johns Hopkins University (author: Daniel Povey)
+#           2022  Zhengyang Chen (chenzhengyang117@gmail.com)
+# Apache 2.0
+
+from __future__ import print_function
+import argparse
+from random import randint
+import sys
+from collections import defaultdict
+
+parser = argparse.ArgumentParser(description="""
+This script, called from data/utils/combine_short_segments.sh, chooses consecutive
+utterances to concatenate that will satisfy the minimum segment length.  It uses the
+--spk2utt file to ensure that utterances from the same speaker are preferentially
+combined (as far as possible while respecting the minimum segment length).
+If it has to combine utterances across different speakers in order to satisfy the
+duration constraint, it will assign the combined utterances to the speaker which
+contributed the most to the duration of the combined utterances.
+
+
+The utt2uts output of this program is a map from new
+utterance-id to a list of old utterance-ids, so for example if the inputs were
+utt1, utt2 and utt3, and utterances 2 and 3 were combined, the output might look
+like:
+utt1 utt1
+utt2-combine2 utt2 utt3
+The utt2spk output of this program assigns utterances to the speakers of the input;
+in the (hopefully rare) case where utterances were combined across speakers, it
+will assign the utterance to whichever of the original speakers contributed the most
+to the grouped utterance.
+""")
+
+parser.add_argument("--min-duration",
+                    type=float,
+                    default=1.55,
+                    help="Minimum utterance duration")
+parser.add_argument(
+    "--merge-within-speakers-only",
+    type=str,
+    default='false',
+    choices=['true', 'false'],
+    help="If true, utterances are only combined from the same speaker."
+    "It may be useful for the speaker recognition task."
+    "If false, utterances are preferentially combined from the same speaker,"
+    "and then combined across different speakers.")
+parser.add_argument(
+    "spk2utt_in",
+    type=str,
+    metavar="<spk2utt-in>",
+    help="Filename of [input] speaker to utterance map needed "
+    "because this script tries to merge utterances from the "
+    "same speaker as much as possible, and also needs to produce"
+    "an output utt2spk map.")
+parser.add_argument("utt2dur_in",
+                    type=str,
+                    metavar="<utt2dur-in>",
+                    help="Filename of [input] utterance-to-duration map, "
+                    "with lines like 'utt1 1.23'.")
+parser.add_argument(
+    "utt2utts_out",
+    type=str,
+    metavar="<utt2utts-out>",
+    help="Filename of [output] new-utterance-to-old-utterances map, with lines "
+    "like 'utt1 utt1' or 'utt2-comb2 utt2 utt3'")
+parser.add_argument(
+    "utt2spk_out",
+    type=str,
+    metavar="<utt2spk-out>",
+    help="Filename of [output] utt2spk map, which maps new utterances to "
+    "original speakers.  If utterances were combined across speakers, "
+    "we map the new utterance to the speaker that contributed the most to them."
+)
+parser.add_argument(
+    "utt2dur_out",
+    type=str,
+    metavar="<utt2spk-out>",
+    help="Filename of [output] utt2dur map, which is just the summations of "
+    "the durations of the source utterances.")
+
+args = parser.parse_args()
+
+
+# This LessThan is designed to be impervious to roundoff effects in cases where
+# numbers are really always separated by a distance >> 1.0e-05.  It will return
+# false if x and y are almost identical, differing only by roundoff effects.
+def LessThan(x, y):
+    return x < y - 1.0e-5
+
+
+# This function implements the core of the utterance-combination code.
+# The input 'durations' is a list of durations, which must all be
+# >=0.0  This function tries to combine consecutive indexes
+# into groups such that for each group, the total duration is at
+# least 'min_duration'.  It returns a list of (start,end) indexes.
+# For example, CombineList(0.1, [5.0,6.0,7.0]) would return
+# [ (0,1), (1,2), (2,3) ] because no combination is necessary; each
+# returned pair represents a singleton group.
+# Or CombineList(1.0, [0.5, 0.6, 0.7]) would return
+# [ (0,3) ].
+# Or CombineList(1.0, [0.5, 0.6, 1.7]) would return
+# [ (0,2), (2,3) ].
+# Note: if sum(durations) < min_duration, this function will
+# return everything in one group but of course the sum of durations
+# will be less than the total.
+def CombineList(min_duration, durations):
+    assert min_duration >= 0.0 and min(durations) > 0.0
+
+    num_utts = len(durations)
+
+    # for each utterance-index i, group_start[i] gives us the
+    # start-index of the group of utterances of which it's currently
+    # a member.
+    group_start = list(range(num_utts))
+    # if utterance-index i currently corresponds to the start of a group
+    # of utterances, then group_durations[i] is the total duration of
+    # that utterance-group, otherwise undefined.
+    group_durations = list(durations)
+    # if utterance-index i currently corresponds to the start of a group
+    # of utterances, then group_end[i] is the end-index
+    # (i.e. last index plus one
+    # of that utterance-group, otherwise undefined.
+    group_end = [x + 1 for x in range(num_utts)]
+
+    queue = [
+        i for i in range(num_utts)
+        if LessThan(group_durations[i], min_duration)
+    ]
+
+    while len(queue) > 0:
+        i = queue.pop()
+        if group_start[i] != i or not LessThan(group_durations[i],
+                                               min_duration):
+            # this group no longer exists or already has at least the
+            # minimum duration.
+            continue
+        this_dur = group_durations[i]
+        # left_dur is the duration of the group to the left of this group,
+        # or 0.0 if there is no such group.
+        left_dur = group_durations[group_start[i - 1]] if i > 0 else 0.0
+        # right_dur is the duration of the group to the right of this group,
+        # or 0.0 if there is no such group.
+        right_dur = group_durations[
+            group_end[i]] if group_end[i] < num_utts else 0.0
+
+        if left_dur == 0.0 and right_dur == 0.0:
+            # there is only one group.  Nothing more to merge; break
+            assert group_start[i] == 0 and group_end[i] == num_utts
+            break
+        # work out whether to combine left or right,
+        # by means of the combine_left variable [ True or False ]
+        if left_dur == 0.0:
+            combine_left = False
+        elif right_dur == 0.0 or LessThan(min_duration, right_dur):
+            combine_left = True
+        elif LessThan(left_dur + this_dur, min_duration):
+            # combining left would still be below the minimum duration->
+            # combine right... if it's above the min duration then good;
+            # otherwise it still doesn't really matter so we might as well
+            # pick one.
+            combine_left = False
+        elif LessThan(right_dur + this_dur, min_duration):
+            # combining right would still be below the minimum duration,
+            # and combining left would be >= the min duration (else we wouldn't
+            # have reached this line) -> combine left.
+            combine_left = True
+        elif LessThan(left_dur, right_dur):
+            # if we reached here then combining either way would take us >= the
+            # minimum duration; but if left_dur < right_dur then we combine left
+            # because that would give us more evenly sized segments.
+            combine_left = True
+        else:
+            # if we reached here then combining either way would take us >= the
+            # minimum duration; but  left_dur >= right_dur, so we combine right
+            # because that would give us more evenly sized segments.
+            combine_left = False
+
+        if combine_left:
+            assert left_dur != 0.0
+            new_group_start = group_start[i - 1]
+            group_end[new_group_start] = group_end[i]
+            for j in range(group_start[i], group_end[i]):
+                group_start[j] = new_group_start
+                group_durations[new_group_start] += durations[j]
+            # note: there is no need to add group_durations[new_group_start] to
+            # the queue even if it is still below the minimum length, because it
+            # would have previously had to have been below the minimum length,
+            # therefore it would already be in the queue.
+        else:
+            assert right_dur != 0.0
+            # group start doesn't change, group end changes.
+            old_group_end = group_end[i]
+            new_group_end = group_end[old_group_end]
+            group_end[i] = new_group_end
+            for j in range(old_group_end, new_group_end):
+                group_durations[i] += durations[j]
+                group_start[j] = i
+            if LessThan(group_durations[i], min_duration):
+                # the group starting at i is still below the minimum length, so
+                # we need to put it back on the queue.
+                queue.append(i)
+
+    ans = []
+    cur_group_start = 0
+    while cur_group_start < num_utts:
+        ans.append((cur_group_start, group_end[cur_group_start]))
+        cur_group_start = group_end[cur_group_start]
+    return ans
+
+
+def SelfTest():
+    assert CombineList(0.1, [5.0, 6.0, 7.0]) == [(0, 1), (1, 2), (2, 3)]
+    assert CombineList(0.5, [0.1, 6.0, 7.0]) == [(0, 2), (2, 3)]
+    assert CombineList(0.5, [6.0, 7.0, 0.1]) == [(0, 1), (1, 3)]
+    # in the two examples below, it combines with the shorter one if both would
+    # be above min-dur.
+    assert CombineList(0.5, [6.0, 0.1, 7.0]) == [(0, 2), (2, 3)]
+    assert CombineList(0.5, [7.0, 0.1, 6.0]) == [(0, 2), (2, 3)]
+    # in the example below, it combines with whichever one would
+    # take it above the min-dur, if there is only one such.
+    # note, it tests the 0.1 first as the queue is popped from the end.
+    assert CombineList(1.0, [1.0, 0.5, 0.1, 6.0]) == [(0, 3), (3, 4)]
+
+    for x in range(100):
+        min_duration = 0.05
+        num_utts = randint(1, 15)
+        durations = []
+        for i in range(num_utts):
+            durations.append(0.01 * randint(1, 10))
+        ranges = CombineList(min_duration, durations)
+        if len(ranges
+               ) > 1:  # check that each range's duration is >= min_duration
+            for j in range(len(ranges)):
+                (start, end) = ranges[j]
+                this_dur = sum([durations[k] for k in range(start, end)])
+                assert not LessThan(this_dur, min_duration)
+
+        # check that the list returned is not affected by very tiny differences
+        # in the inputs.
+        durations2 = list(durations)
+        for i in range(len(durations2)):
+            durations2[i] += 1.0e-07 * randint(-5, 5)
+        ranges2 = CombineList(min_duration, durations2)
+        assert ranges2 == ranges
+
+
+# This function figures out the grouping of utterances.
+# The input is:
+# 'min_duration' which is the minimum utterance length in seconds.
+# 'merge_within_speakers_only' which is a ['true', 'false'] choice.
+# If true, then utterances are only combined if they belong to the same speaker.
+# 'spk2utt' which is a list of pairs (speaker-id, [list-of-utterances])
+# 'utt2dur' which is a dict from utterance-id to duration (as a float)
+# It returns a lists of lists of utterances; each list corresponds to
+# a group, e.g.
+# [ ['utt1'], ['utt2', 'utt3'] ]
+def GetUtteranceGroups(min_duration, merge_within_speakers_only, spk2utt,
+                       utt2dur):
+    # utt_groups will be a list of lists of utterance-ids formed from the
+    # first pass of combination.
+    utt_groups = []
+    # group_durations will be the durations of the corresponding elements of
+    # 'utt_groups'.
+    group_durations = []
+
+    # This block calls CombineList for the utterances of each speaker
+    # separately, in the 'first pass' of combination.
+    for i in range(len(spk2utt)):
+        (spk, utts) = spk2utt[i]
+        durations = []  # durations for this group of utts.
+        for utt in utts:
+            try:
+                durations.append(utt2dur[utt])
+            except Exception:
+                sys.exit("choose_utts_to_combine.py: no duration available "
+                         "in utt2dur file {0} for utterance {1}".format(
+                             args.utt2dur_in, utt))
+        ranges = CombineList(min_duration, durations)
+        for start, end in ranges:  # each element of 'ranges' is a 2-tuple (start, end)
+            utt_groups.append([utts[i] for i in range(start, end)])
+            group_durations.append(
+                sum([durations[i] for i in range(start, end)]))
+
+    old_dur_sum = sum(utt2dur.values())
+    new_dur_sum = sum(group_durations)
+    if abs(old_dur_sum - new_dur_sum) > 0.0001 * old_dur_sum:
+        print("choose_utts_to_combine.py: large difference in total "
+              "durations: {0} vs {1} ".format(old_dur_sum, new_dur_sum),
+              file=sys.stderr)
+
+    # Now we combine the groups obtained above, in case we had situations where
+    # the combination of all the utterances of one speaker were still below
+    # the minimum duration.
+    if merge_within_speakers_only == 'true':
+        return utt_groups
+    else:
+        new_utt_groups = []
+        ranges = CombineList(min_duration, group_durations)
+        for start, end in ranges:
+            # the following code is destructive of 'utt_groups' but it doesn't
+            # matter.
+            this_group = utt_groups[start]
+            for i in range(start + 1, end):
+                this_group += utt_groups[i]
+            new_utt_groups.append(this_group)
+        print(
+            "choose_utts_to_combine.py: combined {0} utterances to {1} utterances "
+            "while respecting speaker boundaries, and then to {2} utterances "
+            "with merging across speaker boundaries.".format(
+                len(utt2dur), len(utt_groups), len(new_utt_groups)),
+            file=sys.stderr)
+        return new_utt_groups
+
+
+SelfTest()
+
+if args.min_duration < 0.0:
+    print("choose_utts_to_combine.py: bad minium duration {0}".format(
+        args.min_duration))
+
+# spk2utt is a list of 2-tuples (speaker-id, [list-of-utterances])
+spk2utt = []
+# utt2spk is a dict from speaker-id to utternace-id.
+utt2spk = dict()
+try:
+    f = open(args.spk2utt_in)
+except Exception:
+    sys.exit("choose_utts_to_combine.py: error opening --spk2utt={0}".format(
+        args.spk2utt_in))
+while True:
+    line = f.readline()
+    if line == '':
+        break
+    a = line.split()
+    if len(a) < 2:
+        sys.exit("choose_utts_to_combine.py: bad line in spk2utt file: " +
+                 line)
+    spk = a[0]
+    utts = a[1:]
+    spk2utt.append((spk, utts))
+    for utt in utts:
+        if utt in utt2spk:
+            sys.exit(
+                "choose_utts_to_combine.py: utterance {0} is listed more than once"
+                "in the spk2utt file {1}".format(utt, args.spk2utt_in))
+        utt2spk[utt] = spk
+f.close()
+
+# utt2dur is a dict from utterance-id (as a string) to
+# duration in seconds (as a float)
+utt2dur = dict()
+try:
+    f = open(args.utt2dur_in)
+except Exception:
+    sys.exit(
+        "choose_utts_to_combine.py: error opening utt2dur file {0}".format(
+            args.utt2dur_in))
+while True:
+    line = f.readline()
+    if line == '':
+        break
+    try:
+        [utt, dur] = line.split()
+        dur = float(dur)
+        utt2dur[utt] = dur
+    except Exception:
+        sys.exit(
+            "choose_utts_to_combine.py: bad line in utt2dur file {0}: {1}".
+            format(args.utt2dur_in, line))
+
+utt_groups = GetUtteranceGroups(args.min_duration,
+                                args.merge_within_speakers_only, spk2utt,
+                                utt2dur)
+
+# set utt_group names to an array like [ 'utt1', 'utt2-comb2', 'utt4', ... ]
+utt_group_names = [
+    group[0] if len(group) == 1 else "{0}-comb{1}".format(
+        group[0], len(group)) for group in utt_groups
+]
+
+# write the utt2utts file.
+try:
+    with open(args.utt2utts_out, 'w') as f:
+        for i in range(len(utt_groups)):
+            print(utt_group_names[i], ' '.join(utt_groups[i]), file=f)
+except Exception as e:
+    sys.exit("choose_utts_to_combine.py: exception writing to "
+             "<utt2utts-out>={0}: {1}".format(args.utt2utts_out, str(e)))
+
+# write the utt2spk file.
+try:
+    with open(args.utt2spk_out, 'w') as f:
+        for i in range(len(utt_groups)):
+            utt_group = utt_groups[i]
+            spk_list = [utt2spk[utt] for utt in utt_group]
+            if spk_list == [spk_list[0]] * len(utt_group):
+                spk = spk_list[0]
+            else:
+                spk2dur = defaultdict(float)
+                # spk2dur is a map from the speaker-id to the duration
+                # within this
+                # utt, that it comprises.
+                for utt in utt_group:
+                    spk2dur[utt2spk[utt]] += utt2dur[utt]
+                # the following code, which picks the speaker that contributed
+                # the most to the duration of this utterance, is a little
+                # complex because we want to break ties in a deterministic way
+                # picking the earlier spaker in case of a tied duration.
+                longest_spk_dur = -1.0
+                spk = None
+                for this_spk in sorted(spk2dur.keys()):
+                    if LessThan(longest_spk_dur, spk2dur[this_spk]):
+                        longest_spk_dur = spk2dur[this_spk]
+                        spk = this_spk
+                assert spk is not None
+            new_utt = utt_group_names[i]
+            print(new_utt, spk, file=f)
+except Exception as e:
+    sys.exit("choose_utts_to_combine.py: exception writing to "
+             "<utt2spk-out>={0}: {1}".format(args.utt2spk_out, str(e)))
+
+# write the utt2dur file.
+try:
+    with open(args.utt2dur_out, 'w') as f:
+        for i in range(len(utt_groups)):
+            utt_name = utt_group_names[i]
+            duration = sum([utt2dur[utt] for utt in utt_groups[i]])
+            print(utt_name, duration, file=f)
+except Exception as e:
+    sys.exit("choose_utts_to_combine.py: exception writing to "
+             "<utt2dur-out>={0}: {1}".format(args.utt2dur_out, str(e)))
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_accd_to_utt2utts.py b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_accd_to_utt2utts.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e2e1dd0841076f85a934c9f7671306897ca897e
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_accd_to_utt2utts.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import numpy as np
+import fire
+import soundfile as sf
+import pypeln as pl
+from tqdm import tqdm
+
+
+def store_comb_data_sub_process(tuple_data):
+    '''
+    tuple: (ori_data_dir, store_data_dir, line)
+    line: spk_id/utt-comb spk_id/utt1 spk_id/utt2 ...
+    '''
+    ori_data_dir, store_data_dir, line = tuple_data
+
+    segs = line.strip().split()
+
+    spk_id = segs[0].split('/')[0]
+    store_dir = os.path.join(store_data_dir, spk_id)
+    os.makedirs(store_dir, exist_ok=True)
+
+    store_path = os.path.join(store_data_dir, segs[0] + '.wav')
+
+    data_list = []
+    for utt_name in segs[1:]:
+        utt_path = os.path.join(ori_data_dir, utt_name + '.flac')
+        data, sr = sf.read(utt_path)
+        data_list.append(data)
+
+    data = np.concatenate(data_list)
+
+    sf.write(store_path, data, sr)
+
+    return 0
+
+
+def store_comb_data(ori_data_dir, store_data_dir, utt2utts, num_process=10):
+
+    with open(utt2utts, 'r') as f:
+        lines = f.readlines()
+        lines_num = len(lines)
+
+        ori_data_dir_list = [ori_data_dir] * lines_num
+        store_data_dir_list = [store_data_dir] * lines_num
+
+        t_bar = tqdm(ncols=100, total=lines_num)
+        for _ in pl.process.map(store_comb_data_sub_process,
+                                zip(ori_data_dir_list, store_data_dir_list,
+                                    lines),
+                                workers=num_process,
+                                maxsize=num_process + 1):
+            t_bar.update()
+
+        t_bar.close()
+
+
+if __name__ == "__main__":
+    fire.Fire(store_comb_data)
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_cn1_cn2.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_cn1_cn2.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f5ad527ebae525014dec6f1344c9b3bb2cda25aa
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/comb_cn1_cn2.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cnceleb1_audio_dir=/data_root/CN-Celeb_flac/data/
+cnceleb2_audio_dir=/data_root/CN-Celeb2_flac/data/
+min_duration=5
+get_dur_nj=60
+statistics_dir=statistics
+store_data_dir=new_data
+
+. tools/parse_options.sh
+set -e
+
+mkdir -p $statistics_dir
+
+
+# combine the short audios for Cnceleb2
+cnceleb2_audio_dir=`realpath $cnceleb2_audio_dir`
+# get the paths of all the audio files
+find $cnceleb2_audio_dir -name "*.flac" | sort > ${statistics_dir}/cnceleb2_audio_path_list
+echo "combine audios for cnceleb2"
+bash local/combine_utt.sh --stage 0 \
+                    --ori_audio_dir ${cnceleb2_audio_dir} \
+                    --new_audio_dir ${store_data_dir}/CN-Celeb2_wav/data \
+                    --data_statistics_dir ${statistics_dir}/cnceleb2 \
+                    --audio_path_list ${statistics_dir}/cnceleb2_audio_path_list \
+                    --min_duration ${min_duration} \
+                    --get_dur_nj ${get_dur_nj}
+
+
+# combine the short audios for Cnceleb1
+cnceleb1_audio_dir=`realpath $cnceleb1_audio_dir`
+# get the paths of all the audio files
+find $cnceleb1_audio_dir -name "*.flac" | awk -F/ '{if($(NF-1)<"id00800"){print $0}}' | sort > ${statistics_dir}/cnceleb1_audio_path_list
+echo "combine audios for cnceleb1_dev"
+bash local/combine_utt.sh --stage 0 \
+                    --ori_audio_dir ${cnceleb1_audio_dir} \
+                    --new_audio_dir ${store_data_dir}/CN-Celeb_wav/data \
+                    --data_statistics_dir ${statistics_dir}/cnceleb1 \
+                    --audio_path_list ${statistics_dir}/cnceleb1_audio_path_list \
+                    --min_duration ${min_duration} \
+                    --get_dur_nj ${get_dur_nj}
+
+# process the remaining flac data of cnceleb1 to wav data
+find $cnceleb1_audio_dir -name "*.flac" | awk -F/ '{if($(NF-1)>="id00800"){print $0}}' | sort > ${statistics_dir}/cnceleb1_eval_audio_path_list
+find $cnceleb1_audio_dir/../eval -name "*.flac" | sort > ${statistics_dir}/cnceleb1_enroll_audio_path_list
+awk -F '[./]' '{print $(NF-2)"/"$(NF-1)" "$(NF-2)"/"$(NF-1)}' ${statistics_dir}/cnceleb1_eval_audio_path_list > ${statistics_dir}/cnceleb1_eval_utt2utts
+awk -F '[./]' '{print $(NF-2)"/"$(NF-1)" "$(NF-2)"/"$(NF-1)}' ${statistics_dir}/cnceleb1_enroll_audio_path_list > ${statistics_dir}/cnceleb1_enroll_utt2utts
+echo "combine audios for cnceleb1_eval"
+python local/comb_accd_to_utt2utts.py --ori_data_dir ${cnceleb1_audio_dir} \
+                                        --store_data_dir ${store_data_dir}/CN-Celeb_wav/data \
+                                        --utt2utts ${statistics_dir}/cnceleb1_eval_utt2utts \
+                                        --num_process 40
+echo "combine audios for cnceleb1_enroll"
+python local/comb_accd_to_utt2utts.py --ori_data_dir ${cnceleb1_audio_dir}/../eval \
+                                        --store_data_dir ${store_data_dir}/CN-Celeb_wav/eval \
+                                        --utt2utts ${statistics_dir}/cnceleb1_enroll_utt2utts \
+                                        --num_process 40
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/combine_utt.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/combine_utt.sh
new file mode 100644
index 0000000000000000000000000000000000000000..05ea06e5827a1c60630a106f75c6e40118ced923
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/combine_utt.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage=0
+ori_audio_dir=/tmp_data_dir
+new_audio_dir=new_data
+data_statistics_dir=statistics
+audio_path_list=''
+min_duration=5
+get_dur_nj=40
+
+. tools/parse_options.sh
+set -e
+
+statistics_dir=$data_statistics_dir/ori_stat
+comb_statistics_dir=$data_statistics_dir/comb_stat
+mkdir -p $statistics_dir
+mkdir -p $comb_statistics_dir
+
+
+if [ $stage -le 0 ]; then
+    awk -F '[./]' '{print $(NF-2)"/"$(NF-1)" "$0}' ${audio_path_list} > ${statistics_dir}/wav.scp
+    awk '{print $1}' ${statistics_dir}/wav.scp > ${statistics_dir}/utt
+    echo "Calcualte the duration for each audio file, this may take for a while..."
+    tools/wav_to_duration.sh --nj ${get_dur_nj} ${statistics_dir}/wav.scp ${statistics_dir}/utt2dur
+    echo "Calcualte the duration done!"
+    # here the spk represents speaker and genre
+    awk -F- '{print $0,$1}' ${statistics_dir}/utt > ${statistics_dir}/utt2spk
+    tools/utt2spk_to_spk2utt.pl ${statistics_dir}/utt2spk > ${statistics_dir}/spk2utt
+fi
+
+if [ $stage -le 1 ]; then
+    python local/choose_utts_to_combine.py --min-duration=${min_duration} \
+        --merge-within-speakers-only=true \
+        ${statistics_dir}/spk2utt ${statistics_dir}/utt2dur ${comb_statistics_dir}/utt2utts ${comb_statistics_dir}/utt2spk ${comb_statistics_dir}/utt2dur
+fi
+
+if [ $stage -le 2 ]; then
+    python local/comb_accd_to_utt2utts.py --ori_data_dir ${ori_audio_dir} --store_data_dir ${new_audio_dir} --utt2utts ${comb_statistics_dir}/utt2utts --num_process 40
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/download_data.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/download_data.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5dc11d439852ffdd79911517aee1354bade296bb
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/download_data.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Hongji Wang (jijijiang77@gmail.com)
+#               2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+download_dir=data/download_data
+
+. tools/parse_options.sh || exit 1
+
+[ ! -d ${download_dir} ] && mkdir -p ${download_dir}
+
+if [ ! -f ${download_dir}/musan.tar.gz ]; then
+  echo "Downloading musan.tar.gz ..."
+  wget --no-check-certificate https://openslr.elda.org/resources/17/musan.tar.gz -P ${download_dir}
+  md5=$(md5sum ${download_dir}/musan.tar.gz | awk '{print $1}')
+  [ $md5 != "0c472d4fc0c5141eca47ad1ffeb2a7df" ] && echo "Wrong md5sum of musan.tar.gz" && exit 1
+fi
+
+if [ ! -f ${download_dir}/rirs_noises.zip ]; then
+  echo "Downloading rirs_noises.zip ..."
+  wget --no-check-certificate https://us.openslr.org/resources/28/rirs_noises.zip -P ${download_dir}
+  md5=$(md5sum ${download_dir}/rirs_noises.zip | awk '{print $1}')
+  [ $md5 != "e6f48e257286e05de56413b4779d8ffb" ] && echo "Wrong md5sum of rirs_noises.zip" && exit 1
+fi
+
+if [ ! -f ${download_dir}/cn-celeb_v2.tar.gz ]; then
+  echo "Downloading cn-celeb_v2.tar.gz ..."
+  wget --no-check-certificate https://www.openslr.org/resources/82/cn-celeb_v2.tar.gz -P ${download_dir}
+  md5=$(md5sum ${download_dir}/cn-celeb_v2.tar.gz | awk '{print $1}')
+  [ $md5 != "7ab1b214028a7439e26608b2d5a0336c" ] && echo "Wrong md5sum of cn-celeb_v2.tar.gz" && exit 1
+fi
+
+if [ ! -f ${download_dir}/cn-celeb2_v2.tar.gz ]; then
+  echo "Downloading cn-celeb2_v2.tar.gz ..."
+  for part in a b c; do
+    wget --no-check-certificate https://www.openslr.org/resources/82/cn-celeb2_v2.tar.gza${part} -P ${download_dir} &
+  done
+  wait
+  cat ${download_dir}/cn-celeb2_v2.tar.gza* >${download_dir}/cn-celeb2_v2.tar.gz
+  md5=$(md5sum ${download_dir}/cn-celeb2_v2.tar.gz | awk '{print $1}')
+  [ $md5 != "55c47cf0b6d0bf793e88bf79d5dfc660" ] && echo "Wrong md5sum of cn-celeb2_v2.tar.gz" && exit 1
+fi
+
+echo "Download success !!!"
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/extract_cnc.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/extract_cnc.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c0260350ef4d22fb0bd8aaaa5d67ea72f71a7cc3
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/extract_cnc.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# coding:utf-8
+
+# Copyright (c) 2022 Hongji Wang (jijijiang77@gmail.com)
+#               2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exp_dir=''
+model_path=''
+nj=4
+gpus="[0,1]"
+data_type="shard/raw"  # shard/raw
+data=data
+
+. tools/parse_options.sh
+set -e
+
+data_name_array=("cnceleb_train" "eval")
+data_list_path_array=("${data}/cnceleb_train/${data_type}.list" "${data}/eval/${data_type}.list")
+data_scp_path_array=("${data}/cnceleb_train/wav.scp" "${data}/eval/wav.scp")
+nj_array=($nj $nj)
+batch_size_array=(16 1) # batch_size of test set must be 1 !!!
+num_workers_array=(4 1)
+count=${#data_name_array[@]}
+
+for i in $(seq 0 $(($count - 1))); do
+  wavs_num=$(wc -l ${data_scp_path_array[$i]} | awk '{print $1}')
+  bash tools/extract_embedding.sh --exp_dir ${exp_dir} \
+    --model_path $model_path \
+    --data_type ${data_type} \
+    --data_list ${data_list_path_array[$i]} \
+    --wavs_num ${wavs_num} \
+    --store_dir ${data_name_array[$i]} \
+    --batch_size ${batch_size_array[$i]} \
+    --num_workers ${num_workers_array[$i]} \
+    --nj ${nj_array[$i]} \
+    --gpus $gpus &
+done
+
+wait
+
+echo "Embedding dir is (${exp_dir}/embeddings)."
+
+echo "mean vector of enroll"
+python tools/vector_mean.py \
+  --spk2utt ${data}/eval/enroll.map \
+  --xvector_scp $exp_dir/embeddings/eval/xvector.scp \
+  --spk_xvector_ark $exp_dir/embeddings/eval/enroll_spk_xvector.ark
+
+cat $exp_dir/embeddings/eval/enroll_spk_xvector.scp >> $exp_dir/embeddings/eval/xvector.scp
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/flac2wav.py b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/flac2wav.py
new file mode 100644
index 0000000000000000000000000000000000000000..64e936454a40ff562b8869e3c568dcd914ea827c
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/flac2wav.py
@@ -0,0 +1,68 @@
+# Copyright (c) 2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import argparse
+
+from tqdm import tqdm
+from multiprocessing import Pool
+
+
+def find_all_wav(dirname, extension='flac'):
+    if dirname[-1] != os.sep:
+        dirname += os.sep
+    old_name = dirname.split('/')[-2]
+    new_name = old_name.replace("flac", "wav")
+    print("find {}, waiting ...".format(extension))
+    flac2wav = []
+    for root, _, filenames in tqdm(os.walk(dirname, followlinks=True)):
+        wav_files = [f for f in filenames if f.endswith(extension)]
+        if len(wav_files) > 0:
+            new_root = root.replace(old_name, new_name)
+            if not os.path.exists(new_root):
+                print(new_root)
+                os.makedirs(new_root)
+            for _wav in wav_files:
+                old_path = os.path.join(root, _wav)
+                new_path = os.path.join(new_root,
+                                        _wav.replace(".flac", ".wav"))
+                flac2wav.append([old_path, new_path])
+    return flac2wav
+
+
+def flac2wav_main(flac_list):
+    flac_path, wav_path = flac_list[0], flac_list[1]
+    print("convert {}".format(flac_path))
+    if os.path.exists(wav_path):
+        os.remove(wav_path)
+    cmd = "sox -t flac {} -t wav -r 16k -b 16 {} channels 1".format(
+        flac_path, wav_path)
+    os.system(cmd)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataset_dir',
+                        type=str,
+                        default="data",
+                        help="dataset dir")
+    parser.add_argument('--nj',
+                        type=int,
+                        default=8,
+                        help="number of processes")
+    args = parser.parse_args()
+
+    flac_list = find_all_wav(args.dataset_dir)
+    with Pool(args.nj) as p:
+        p.map(flac2wav_main, flac_list)
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/format_trials_cnceleb.py b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/format_trials_cnceleb.py
new file mode 100644
index 0000000000000000000000000000000000000000..9da805dbd59c01f0d2acdfe1f77cceb8d522d05b
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/format_trials_cnceleb.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+import os
+import argparse
+import numpy as np
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cnceleb_root',
+                        help='cnceleb dir',
+                        type=str,
+                        default="CN-Celeb")
+    parser.add_argument('--dst_trl_path',
+                        help='output trial path',
+                        type=str,
+                        default="new.trials")
+    args = parser.parse_args()
+
+    enroll_lst_path = os.path.join(args.cnceleb_root, "eval/lists/enroll.lst")
+    raw_trl_path = os.path.join(args.cnceleb_root, "eval/lists/trials.lst")
+
+    spk2wav_mapping = {}
+    enroll_lst = np.loadtxt(enroll_lst_path, str)
+    for item in enroll_lst:
+        spk2wav_mapping[item[0]] = item[1]
+    trials = np.loadtxt(raw_trl_path, str)
+
+    with open(args.dst_trl_path, "w") as f:
+        for item in trials:
+            enroll_path = spk2wav_mapping[item[0]]
+            test_path = item[1]
+            label = "target" if item[2] == '1' else "nontarget"
+            f.write("{} {} {}\n".format(enroll_path, test_path, label))
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/prepare_data.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/prepare_data.sh
new file mode 100644
index 0000000000000000000000000000000000000000..310f0aa9862c857c7e4f349a15305147cff16ec1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/prepare_data.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# coding:utf-8
+
+# Copyright (c) 2022 Hongji Wang (jijijiang77@gmail.com)
+#               2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#               2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage=-1
+stop_stage=-1
+combine_short_audio=1
+data=data
+
+. tools/parse_options.sh || exit 1
+
+data=`realpath ${data}`
+download_dir=${data}/download_data
+rawdata_dir=${data}/raw_data
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "Download musan.tar.gz, rirs_noises.zip, cn-celeb_v2.tar.gz and cn-celeb2_v2.tar.gz."
+  echo "This may take a long time. Thus we recommand you to download all archives above in your own way first."
+
+  ./local/download_data.sh --download_dir ${download_dir}
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "Decompress all archives ..."
+  echo "This could take some time ..."
+
+  for archive in musan.tar.gz rirs_noises.zip cn-celeb_v2.tar.gz cn-celeb2_v2.tar.gz; do
+    [ ! -f ${download_dir}/$archive ] && echo "Archive $archive not exists !!!" && exit 1
+  done
+  [ ! -d ${rawdata_dir} ] && mkdir -p ${rawdata_dir}
+
+  if [ ! -d ${rawdata_dir}/musan ]; then
+    tar -xzvf ${download_dir}/musan.tar.gz -C ${rawdata_dir}
+  fi
+
+  if [ ! -d ${rawdata_dir}/RIRS_NOISES ]; then
+    unzip ${download_dir}/rirs_noises.zip -d ${rawdata_dir}
+  fi
+
+  if [ ! -d ${rawdata_dir}/CN-Celeb_flac ]; then
+    tar -xzvf ${download_dir}/cn-celeb_v2.tar.gz -C ${rawdata_dir}
+  fi
+
+  if [ ! -d ${rawdata_dir}/CN-Celeb2_flac ]; then
+    tar -xzvf ${download_dir}/cn-celeb2_v2.tar.gz -C ${rawdata_dir}
+  fi
+
+  echo "Decompress success !!!"
+fi
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  if [ ${combine_short_audio} -eq 1 ];then
+    echo "combine short audios and convert flac to wav ..."
+    bash local/comb_cn1_cn2.sh --cnceleb1_audio_dir ${rawdata_dir}/CN-Celeb_flac/data/ \
+                                --cnceleb2_audio_dir ${rawdata_dir}/CN-Celeb2_flac/data/ \
+                                --min_duration 5 \
+                                --get_dur_nj 60 \
+                                --statistics_dir ${data}/statistics \
+                                --store_data_dir ${rawdata_dir}
+    echo "convert success"
+  else
+    echo "convert flac to wav ..."
+    python local/flac2wav.py \
+        --dataset_dir ${rawdata_dir}/CN-Celeb_flac \
+        --nj 16
+
+    python local/flac2wav.py \
+        --dataset_dir ${rawdata_dir}/CN-Celeb2_flac \
+        --nj 16
+    echo "convert success"
+  fi
+fi
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  echo "Prepare wav.scp for each dataset ..."
+  export LC_ALL=C # kaldi config
+
+  mkdir -p ${data}/musan ${data}/rirs ${data}/cnceleb_train ${data}/eval
+  # musan
+  find ${rawdata_dir}/musan -name "*.wav" | awk -F"/" '{print $(NF-2)"/"$(NF-1)"/"$NF,$0}' >${data}/musan/wav.scp || exit 1;
+  # rirs
+  find ${rawdata_dir}/RIRS_NOISES/simulated_rirs -name "*.wav" | awk -F"/" '{print $(NF-2)"/"$(NF-1)"/"$NF,$0}' >${data}/rirs/wav.scp || exit 1;
+
+  echo "Prepare train data including CN-Celeb_wav/dev and CN-Celeb2_wav ..."
+  [ -f ${data}/cnceleb_train/wav.scp ] && rm ${data}/cnceleb_train/wav.scp
+  for spk in `cat ${rawdata_dir}/CN-Celeb_flac/dev/dev.lst`; do
+    find ${rawdata_dir}/CN-Celeb_wav/data/${spk} -name "*.wav" | \
+      awk -F"/" '{print $(NF-1)"/"$NF,$0}' | sort >>${data}/cnceleb_train/wav.scp
+  done
+
+  for spk in `cat ${rawdata_dir}/CN-Celeb2_flac/spk.lst`; do
+    find ${rawdata_dir}/CN-Celeb2_wav/data/${spk} -name "*.wav" | \
+      awk -F"/" '{print $(NF-1)"/"$NF,$0}' | sort >>${data}/cnceleb_train/wav.scp
+  done
+
+  awk '{print $1}' ${data}/cnceleb_train/wav.scp | awk -F "/" '{print $0,$1}' >${data}/cnceleb_train/utt2spk
+  ./tools/utt2spk_to_spk2utt.pl ${data}/cnceleb_train/utt2spk >${data}/cnceleb_train/spk2utt
+
+  echo "Prepare data for testing ..."
+  find ${rawdata_dir}/CN-Celeb_wav/eval -name "*.wav" | awk -F"/" '{print $(NF-1)"/"$NF,$0}' | sort >${data}/eval/wav.scp
+  awk '{print $1}' ${data}/eval/wav.scp | awk -F "[/-]" '{print $0,$2}' >${data}/eval/utt2spk
+
+  echo "Prepare data for enroll ..."
+  awk '{print $0}' ${rawdata_dir}/CN-Celeb_flac/eval/lists/enroll.map | \
+    awk -v p=${rawdata_dir}/CN-Celeb_wav/data '{for(i=2;i<=NF;i++){print $i, p"/"$i}}' >${data}/eval/enroll.scp
+  cat ${data}/eval/enroll.scp >>${data}/eval/wav.scp
+  awk '{print $1}' ${data}/eval/enroll.scp | awk -F "/" '{print $0,$1"-enroll"}' >>${data}/eval/utt2spk
+  cp ${rawdata_dir}/CN-Celeb_flac/eval/lists/enroll.map ${data}/eval/enroll.map
+
+  echo "Prepare evalution trials ..."
+  mkdir -p ${data}/eval/trials
+  # CNC-Eval-Avg.lst
+  awk '{if($3==0)label="nontarget";else{label="target"}; print $1,$2,label}' ${rawdata_dir}/CN-Celeb_flac/eval/lists/trials.lst >${data}/eval/trials/CNC-Eval-Avg.lst
+  # CNC-Eval-Concat.lst
+  python local/format_trials_cnceleb.py \
+    --cnceleb_root ${rawdata_dir}/CN-Celeb_flac \
+    --dst_trl_path ${data}/eval/trials/CNC-Eval-Concat.lst
+
+  echo "Success !!! Now data preparation is done !!!"
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score.sh
new file mode 100644
index 0000000000000000000000000000000000000000..497e28f88950312970fbca25129914f75e434c0d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exp_dir=
+trials="CNC-Eval-Concat.lst CNC-Eval-Avg.lst"
+data=data
+
+stage=-1
+stop_stage=-1
+
+. tools/parse_options.sh
+. path.sh
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "apply cosine scoring ..."
+  mkdir -p ${exp_dir}/scores
+  trials_dir=${data}/eval/trials
+  for x in $trials; do
+    echo $x
+    python wespeaker/bin/score.py \
+      --exp_dir ${exp_dir} \
+      --eval_scp_path ${exp_dir}/embeddings/eval/xvector.scp \
+      --cal_mean True \
+      --cal_mean_dir ${exp_dir}/embeddings/cnceleb_train \
+      ${trials_dir}/${x}
+  done
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "compute metrics (EER/minDCF) ..."
+  scores_dir=${exp_dir}/scores
+  for x in $trials; do
+    python wespeaker/bin/compute_metrics.py \
+        --p_target 0.01 \
+        --c_fa 1 \
+        --c_miss 1 \
+        ${scores_dir}/${x}.score \
+        2>&1 | tee -a ${scores_dir}/cnc_cos_result
+
+    echo "compute DET curve ..."
+    python wespeaker/bin/compute_det.py \
+        ${scores_dir}/${x}.score
+  done
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_calibration.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_calibration.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9e04765dd28c8a51e25df3a8d38bbe311d26dcb9
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_calibration.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#               2024 Zhengyang Chen (chenzhengyang117@gmail.com)
+#               2024 Bing Han (hanbing97@sjtu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+score_norm_method="asnorm"  # asnorm/snorm
+cohort_set=cnceleb_train
+calibration_trial="cn_dev_cali.kaldi"
+top_n=100
+exp_dir=''
+trials="CNC-Eval-Concat.lst CNC-Eval-Avg.lst"
+data=data
+
+stage=-1
+stop_stage=-1
+
+. tools/parse_options.sh
+. path.sh
+
+output_name=${cohort_set}_${score_norm_method}
+[ "${score_norm_method}" == "asnorm" ] && output_name=${output_name}${top_n}
+trials_dir=${data}/eval/trials
+
+if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
+  echo "Score calibration set"
+  # Compute duration
+  if [ ! -f ${data}/cnceleb_train/utt2dur ]; then
+    cat ${data}/statistics/cnceleb2/comb_stat/utt2dur ${data}/statistics/cnceleb1/comb_stat/utt2dur | awk '{print $1".wav", $2}' > ${data}/cnceleb_train/utt2dur
+  fi
+  # generate trial for calibration
+  if [ ! -e ${trials_dir}/${calibration_trial} ]; then
+    python tools/generate_calibration_trial.py --utt2dur ${data}/cnceleb_train/utt2dur --trial_path ${trials_dir}/${calibration_trial} --each_trial_num 20000
+  fi
+
+  python wespeaker/bin/score.py \
+    --exp_dir ${exp_dir} \
+    --eval_scp_path ${exp_dir}/embeddings/cnceleb_train/xvector.scp \
+    --cal_mean True \
+    --cal_mean_dir ${exp_dir}/embeddings/cnceleb_train \
+    ${trials_dir}/${calibration_trial}
+
+  python wespeaker/bin/score_norm.py \
+    --score_norm_method $score_norm_method \
+    --top_n $top_n \
+    --trial_score_file $exp_dir/scores/${calibration_trial}.score \
+    --score_norm_file $exp_dir/scores/${output_name}_${calibration_trial}.score \
+    --cohort_emb_scp ${exp_dir}/embeddings/${cohort_set}/spk_xvector.scp \
+    --eval_emb_scp ${exp_dir}/embeddings/cnceleb_train/xvector.scp \
+    --mean_vec_path ${exp_dir}/embeddings/cnceleb_train/mean_vec.npy
+fi
+
+if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
+  echo "Prepare calibration factors"
+  # gather calibration factor
+  mkdir -p ${exp_dir}/scores/calibration
+  for x in ${calibration_trial} $trials; do
+    python wespeaker/bin/score_calibration.py "gather_calibration_factors" \
+      --wav_dur_scp ${exp_dir}/scores/calibration/utt2dur \
+      --max_dur 20 \
+      --score_norm_file ${exp_dir}/scores/${output_name}_${x}.score \
+      --calibration_factor_file ${exp_dir}/scores/calibration/${output_name}_${x}.calibration \
+      --drop_duration True
+  done
+fi
+
+
+if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
+  echo "Train calibration model"
+  python wespeaker/bin/score_calibration.py "train_calibration_model" \
+    --calibration_factor_file ${exp_dir}/scores/calibration/${output_name}_${calibration_trial}.calibration \
+    --save_model_path ${exp_dir}/scores/calibration/calibration_model.pt
+fi
+
+cali_output_name=cali_${output_name}
+if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then
+  echo "Infer calibration model"
+  for x in ${trials}; do
+    python wespeaker/bin/score_calibration.py "infer_calibration" \
+      --calibration_factor_file ${exp_dir}/scores/calibration/${output_name}_${x}.calibration \
+      --save_model_path ${exp_dir}/scores/calibration/calibration_model.pt \
+      --calibration_score_file ${exp_dir}/scores/${cali_output_name}_${x}.score
+  done
+fi
+
+if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then
+  echo "compute metrics"
+  for x in ${trials}; do
+    scores_dir=${exp_dir}/scores
+    python wespeaker/bin/compute_metrics.py \
+      --p_target 0.01 \
+      --c_fa 1 \
+      --c_miss 1 \
+      ${scores_dir}/${cali_output_name}_${x}.score \
+      2>&1 | tee -a ${scores_dir}/cnc_cali_${score_norm_method}${top_n}_result
+
+    python wespeaker/bin/compute_det.py \
+      ${scores_dir}/${cali_output_name}_${x}.score
+  done
+fi
\ No newline at end of file
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_norm.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_norm.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b3f2e843828ffa6b47baac31101647188d07f5b9
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/local/score_norm.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+score_norm_method="asnorm"  # asnorm/snorm
+cohort_set=cnceleb_train
+top_n=100
+exp_dir=
+trials="CNC-Eval-Concat.lst CNC-Eval-Avg.lst"
+data=data
+
+stage=-1
+stop_stage=-1
+
+. tools/parse_options.sh
+. path.sh
+
+
+if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
+  echo "compute mean xvector"
+  python tools/vector_mean.py \
+    --spk2utt ${data}/${cohort_set}/spk2utt \
+    --xvector_scp $exp_dir/embeddings/${cohort_set}/xvector.scp \
+    --spk_xvector_ark $exp_dir/embeddings/${cohort_set}/spk_xvector.ark
+fi
+
+output_name=${cohort_set}_${score_norm_method}
+[ "${score_norm_method}" == "asnorm" ] && output_name=${output_name}${top_n}
+if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then
+  echo "compute norm score"
+  for x in $trials; do
+    python wespeaker/bin/score_norm.py \
+      --score_norm_method $score_norm_method \
+      --top_n $top_n \
+      --trial_score_file $exp_dir/scores/${x}.score \
+      --score_norm_file $exp_dir/scores/${output_name}_${x}.score \
+      --cohort_emb_scp ${exp_dir}/embeddings/${cohort_set}/spk_xvector.scp \
+      --eval_emb_scp ${exp_dir}/embeddings/eval/xvector.scp \
+      --mean_vec_path ${exp_dir}/embeddings/cnceleb_train/mean_vec.npy
+  done
+fi
+
+if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then
+  echo "compute metrics"
+  for x in ${trials}; do
+    scores_dir=${exp_dir}/scores
+    python wespeaker/bin/compute_metrics.py \
+      --p_target 0.01 \
+      --c_fa 1 \
+      --c_miss 1 \
+      ${scores_dir}/${output_name}_${x}.score \
+      2>&1 | tee -a ${scores_dir}/cnc_${score_norm_method}${top_n}_result
+
+    python wespeaker/bin/compute_det.py \
+      ${scores_dir}/${output_name}_${x}.score
+  done
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/path.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/path.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b90a5154cf4a8f25b9a42851c59393f07dacb350
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/path.sh
@@ -0,0 +1,5 @@
+export PATH=$PWD:$PATH
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/run.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f0f7f21605c297a83497289944329acc72016d12
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v2/run.sh
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+# Copyright 2022 Hongji Wang (jijijiang77@gmail.com)
+#           2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#           2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#           2024 Bing Han (hanbing97@sjtu.edu.cn)
+
+. ./path.sh || exit 1
+
+# multi-node + multi-gpus:
+#   bash run.sh --stage 3 --stop-stage 3 --HOST_NODE_ADDR "xxx.xxx.xxx.xxx:port" --num_nodes num_node
+
+stage=-1
+stop_stage=-1
+
+HOST_NODE_ADDR="localhost:29400"
+num_nodes=1
+job_id=2024
+
+data=data
+data_type="shard"  # shard/raw
+
+config=conf/resnet.yaml
+exp_dir=exp/ResNet34-TSTP-emb256-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150
+gpus="[0,1]"
+num_avg=10
+checkpoint=
+
+trials="CNC-Eval-Concat.lst CNC-Eval-Avg.lst"
+score_norm_method="asnorm"  # asnorm/snorm
+top_n=300
+
+# setup for large margin fine-tuning
+lm_config=conf/resnet_lm.yaml
+
+. tools/parse_options.sh || exit 1
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "Preparing datasets ..."
+  ./local/prepare_data.sh --stage 2 --stop_stage 4 --data ${data}
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "Covert train and test data to ${data_type}..."
+  for dset in cnceleb_train eval; do
+    if [ $data_type == "shard" ]; then
+      python tools/make_shard_list.py --num_utts_per_shard 1000 \
+          --num_threads 16 \
+          --prefix shards \
+          --shuffle \
+          ${data}/$dset/wav.scp ${data}/$dset/utt2spk \
+          ${data}/$dset/shards ${data}/$dset/shard.list
+    else
+      python tools/make_raw_list.py ${data}/$dset/wav.scp \
+          ${data}/$dset/utt2spk ${data}/$dset/raw.list
+    fi
+  done
+  # Convert all musan data to LMDB
+  python tools/make_lmdb.py ${data}/musan/wav.scp ${data}/musan/lmdb
+  # Convert all rirs data to LMDB
+  python tools/make_lmdb.py ${data}/rirs/wav.scp ${data}/rirs/lmdb
+fi
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "Start training ..."
+  num_gpus=$(echo $gpus | awk -F ',' '{print NF}')
+  echo "$0: num_nodes is $num_nodes, proc_per_node is $num_gpus"
+  torchrun --nnodes=$num_nodes --nproc_per_node=$num_gpus \
+           --rdzv_id=$job_id --rdzv_backend="c10d" --rdzv_endpoint=$HOST_NODE_ADDR \
+    wespeaker/bin/train.py --config $config \
+      --exp_dir ${exp_dir} \
+      --gpus $gpus \
+      --num_avg ${num_avg} \
+      --data_type "${data_type}" \
+      --train_data ${data}/cnceleb_train/${data_type}.list \
+      --train_label ${data}/cnceleb_train/utt2spk \
+      --reverb_data ${data}/rirs/lmdb \
+      --noise_data ${data}/musan/lmdb \
+      ${checkpoint:+--checkpoint $checkpoint}
+fi
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  echo "Do model average ..."
+  avg_model=$exp_dir/models/avg_model.pt
+  python wespeaker/bin/average_model.py \
+    --dst_model $avg_model \
+    --src_path $exp_dir/models \
+    --num ${num_avg}
+
+  model_path=$avg_model
+  if [[ $config == *repvgg*.yaml ]]; then
+    echo "convert repvgg model ..."
+    python wespeaker/models/convert_repvgg.py \
+      --config $exp_dir/config.yaml \
+      --load $avg_model \
+      --save $exp_dir/models/convert_model.pt
+    model_path=$exp_dir/models/convert_model.pt
+  fi
+
+  echo "Extract embeddings ..."
+  local/extract_cnc.sh \
+    --exp_dir $exp_dir --model_path $model_path \
+    --nj 4 --gpus $gpus --data_type $data_type --data ${data}
+fi
+
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+  echo "Score ..."
+  local/score.sh \
+    --stage 1 --stop-stage 2 \
+    --exp_dir $exp_dir \
+    --data ${data} \
+    --trials "$trials"
+fi
+
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  echo "Score norm ..."
+  local/score_norm.sh \
+    --stage 1 --stop-stage 3 \
+    --score_norm_method $score_norm_method \
+    --cohort_set cnceleb_train \
+    --top_n $top_n \
+    --exp_dir $exp_dir \
+    --data ${data} \
+    --trials "$trials"
+fi
+
+# ================== Score Calibration ==================
+# It shoule be noted that the score calibration is optio-
+# nal. For CN-Celeb, it will improve the EER but degrade
+# minDCF.
+
+if [ ${stage} -le 7 ] && [ ${stop_stage} -ge 7 ]; then
+  echo "Score calibration ..."
+  local/score_calibration.sh \
+    --stage 1 --stop-stage 5 \
+    --score_norm_method $score_norm_method \
+    --calibration_trial "cn_dev_cali.kaldi" \
+    --cohort_set cnceleb_train \
+    --top_n $top_n \
+    --exp_dir $exp_dir \
+    --data ${data} \
+    --trials "$trials"
+fi
+
+if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
+  echo "Export the best model ..."
+  python wespeaker/bin/export_jit.py \
+    --config $exp_dir/config.yaml \
+    --checkpoint $exp_dir/models/avg_model.pt \
+    --output_file $exp_dir/models/final.zip
+fi
+
+# ================== Large margin fine-tuning ==================
+# for reference: https://arxiv.org/abs/2206.11699
+# It shoule be noted that the large margin fine-tuning
+# is optional. It often be used in speaker verification
+# challenge to further improve performance. This training
+# proces will take longer segment as input and will take
+# up more gpu memory.
+
+if [ ${stage} -le 9 ] && [ ${stop_stage} -ge 9 ]; then
+  echo "Large margin fine-tuning ..."
+  lm_exp_dir=${exp_dir}-LM
+  mkdir -p ${lm_exp_dir}/models
+  # Use the pre-trained average model to initialize the LM training
+  cp ${exp_dir}/models/avg_model.pt ${lm_exp_dir}/models/model_0.pt
+  bash run.sh --stage 3 --stop_stage 8 \
+      --data ${data} \
+      --data_type ${data_type} \
+      --config ${lm_config} \
+      --exp_dir ${lm_exp_dir} \
+      --gpus $gpus \
+      --num_avg 1 \
+      --checkpoint ${lm_exp_dir}/models/model_0.pt \
+      --trials "$trials" \
+      --score_norm_method ${score_norm_method} \
+      --top_n ${top_n}
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/README.md b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..715c4fee9b834445cee9c57eb487c8a99412b542
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/README.md
@@ -0,0 +1,25 @@
+## Fine-tuning Results Based on DINO
+
+* Setup: fbank80, num_frms200, epoch50 (finetune), ArcMargin, aug_prob0.6, speed_perturb (no spec_aug)
+* test_trials: CNC-Eval-Avg.lst
+* These results are obtained by pretraining on different datasets and then finetuning with CNCeleb.
+
+| Model                             | Params  |  FLOPs  |    Pretraining Data    | LM  | AS-Norm   | EER (%)   | minDCF (p=0.01)  |
+| :------------------------------   | :-----: | :-----: | :--------------------: | :-: | :-------: | :-------: | :--------------: |
+| ECAPA_TDNN_GLOB_c1024-ASTP-emb192 | 14.65M  | 2.65 G  |        CNCeleb         | ×   | ×         | 8.217     | 0.439            |
+|                                   |         |         |                        | ×   | √         | 7.508     | 0.378            |
+|                                   |         |         |                        | √   | ×         | 8.093     | 0.423            |
+|                                   |         |         |                        | √   | √         | 7.339     | 0.366            |
+|                                   |         |         | WenetSpeech (filtered) | ×   | ×         | 7.229     | 0.390            |
+|                                   |         |         |                        | ×   | √         | 6.714     | 0.344            |
+|                                   |         |         |                        | √   | ×         | 6.995     | 0.375            |
+|                                   |         |         |                        | √   | √         | 6.474     | 0.331            |
+
+* 🔥 UPDATE 2024.03: We support finetuning DINO-based self-supervised models, which is trained on the WenetSpeech dataset. Pretrained Paper related to the finetuning results:
+    * [WenetSpeech: A 10000+ Hours Multi-domain Mandarin Corpus for Speech Recognition](https://arxiv.org/pdf/2110.03370.pdf)
+    * [Leveraging In-the-wild Data for Effective Self-supervised Pretraining in Speaker Recognition](https://arxiv.org/pdf/2309.11730.pdf)
+
+## Resources
+* [Pre-trained ECAPA-TDNN checkpoints](https://drive.google.com/drive/folders/1XDIUjnKPrvJE5auBWT5CcE4mqcglCwzq?usp=drive_link)
+* [The filtering metadata for wenetspeech](https://drive.google.com/file/d/1UaGuyT1wcKc5g9vRdfIBvLoDRcuOxBlX/view?usp=drive_link)
+
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_ft.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_ft.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6bca1d1aea24cd20140f159101356b9543899ab4
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_ft.yaml
@@ -0,0 +1,82 @@
+### train configuraton
+
+exp_dir: exp/ECAPA_TDNN_GLOB_c1024-ASTP-emb192-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-fineTuneDINO-SGD-epoch50
+gpus: "[0,1]"
+num_avg: 3
+enable_amp: False # whether enable automatic mixed precision training
+
+seed: 42
+num_epochs: 50
+save_epoch_interval: 1 # save model every 5 epochs
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 128
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 200
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ECAPA_TDNN_GLOB_c1024 # ECAPA_TDNN_GLOB_c512, ECAPA_TDNN_GLOB_c1024
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 192
+  pooling_func: "ASTP"
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.0
+  final_margin: 0.2
+  increase_start_epoch: 7
+  fix_start_epoch: 14
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 5.0e-3
+  final_lr: 2.5e-5
+  warm_up_epoch: 2
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_lm.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_lm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5290296217c071d315a5a0017d66b5b2a75cb2f1
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/conf/ecapa_tdnn_lm.yaml
@@ -0,0 +1,88 @@
+### Large margin fine-tuning configuration
+#
+#   The large margin fine-tuning operation is often used in speaker
+#   verification challenge system to further improve the performance.
+#   In this fine-tuning stage, large margin and longer segment will
+#   be used.
+
+exp_dir: exp/ECAPA_TDNN_GLOB_c512-ASTP-emb192-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-SGD-epoch150-LM
+gpus: "[0,1]"
+num_avg: 1
+enable_amp: False # whether enable automatic mixed precision training
+do_lm: True
+
+seed: 42
+num_epochs: 5
+save_epoch_interval: 1 # save model per epoch
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 64
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 0
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 2500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 800
+  resample_rate: 16000
+  speed_perturb: True
+  num_frms: 600
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 80
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ECAPA_TDNN_GLOB_c1024 # ECAPA_TDNN_GLOB_c512, ECAPA_TDNN_GLOB_c1024
+model_init: null
+model_args:
+  feat_dim: 80
+  embed_dim: 192
+  pooling_func: "ASTP"
+projection_args:
+  project_type: "arc_margin" # add_margin, arc_margin, sphere, softmax
+  scale: 32.0
+  easy_margin: False
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.5
+  final_margin: 0.5
+  increase_start_epoch: 1
+  fix_start_epoch: 1
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 1.0e-4
+  final_lr: 2.5e-5
+  warm_up_epoch: 1
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/path.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/path.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b90a5154cf4a8f25b9a42851c59393f07dacb350
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/path.sh
@@ -0,0 +1,5 @@
+export PATH=$PWD:$PATH
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/run.sh b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..79e9328f825ac7dc0d553d2c5be3be085a4193ea
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/cnceleb/v3_finetune/run.sh
@@ -0,0 +1,162 @@
+#!/bin/bash
+
+# Copyright 2022 Hongji Wang (jijijiang77@gmail.com)
+#           2022 Chengdong Liang (liangchengdong@mail.nwpu.edu.cn)
+#           2022 Zhengyang Chen (chenzhengyang117@gmail.com)
+#           2024 Qibing Bai (kibing.bai@gmail.com)
+
+. ./path.sh || exit 1
+
+stage=-1
+stop_stage=-1
+
+HOST_NODE_ADDR="localhost:29400"
+num_nodes=1
+job_id=2024
+
+data=data
+data_type="shard"  # shard/raw
+
+config=conf/ecapa_tdnn_ft.yaml
+exp_dir=exp/ECAPA_TDNN_GLOB_c1024-ASTP-emb192-fbank80-num_frms200-aug0.6-spTrue-saFalse-ArcMargin-fineTuneDINO-SGD-epoch50
+gpus="[0,1]"
+num_avg=3
+checkpoint=
+
+# averaged model after self-supervised pre-training (see v3 for details)
+model_init=path/to/init/model
+
+trials="CNC-Eval-Concat.lst CNC-Eval-Avg.lst"
+score_norm_method="asnorm"  # asnorm/snorm
+top_n=300
+
+# setup for large margin fine-tuning
+lm_config=conf/ecapa_tdnn_lm.yaml
+
+. tools/parse_options.sh || exit 1
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "Preparing datasets ..."
+  ./local/prepare_data.sh --stage 2 --stop_stage 4 --data ${data}
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "Covert train and test data to ${data_type}..."
+  for dset in cnceleb_train eval; do
+    if [ $data_type == "shard" ]; then
+      python tools/make_shard_list.py --num_utts_per_shard 1000 \
+          --num_threads 16 \
+          --prefix shards \
+          --shuffle \
+          ${data}/$dset/wav.scp ${data}/$dset/utt2spk \
+          ${data}/$dset/shards ${data}/$dset/shard.list
+    else
+      python tools/make_raw_list.py ${data}/$dset/wav.scp \
+          ${data}/$dset/utt2spk ${data}/$dset/raw.list
+    fi
+  done
+  # Convert all musan data to LMDB
+  python tools/make_lmdb.py ${data}/musan/wav.scp ${data}/musan/lmdb
+  # Convert all rirs data to LMDB
+  python tools/make_lmdb.py ${data}/rirs/wav.scp ${data}/rirs/lmdb
+fi
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "Start training ..."
+  num_gpus=$(echo $gpus | awk -F ',' '{print NF}')
+  echo "$0: num_nodes is $num_nodes, proc_per_node is $num_gpus"
+  torchrun --nnodes=$num_nodes --nproc_per_node=$num_gpus \
+           --rdzv_id=$job_id --rdzv_backend="c10d" --rdzv_endpoint=$HOST_NODE_ADDR \
+    wespeaker/bin/train.py --config $config \
+      --exp_dir ${exp_dir} \
+      --gpus $gpus \
+      --num_avg ${num_avg} \
+      --data_type "${data_type}" \
+      --train_data ${data}/cnceleb_train/${data_type}.list \
+      --train_label ${data}/cnceleb_train/utt2spk \
+      --reverb_data ${data}/rirs/lmdb \
+      --noise_data ${data}/musan/lmdb \
+      ${model_init:+--model_init $model_init} \
+      ${checkpoint:+--checkpoint $checkpoint}
+fi
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  echo "Do model average ..."
+  avg_model=$exp_dir/models/avg_model.pt
+  python wespeaker/bin/average_model.py \
+    --dst_model $avg_model \
+    --src_path $exp_dir/models \
+    --num ${num_avg}
+
+  model_path=$avg_model
+  if [[ $config == *repvgg*.yaml ]]; then
+    echo "convert repvgg model ..."
+    python wespeaker/models/convert_repvgg.py \
+      --config $exp_dir/config.yaml \
+      --load $avg_model \
+      --save $exp_dir/models/convert_model.pt
+    model_path=$exp_dir/models/convert_model.pt
+  fi
+
+  echo "Extract embeddings ..."
+  local/extract_cnc.sh \
+    --exp_dir $exp_dir --model_path $model_path \
+    --nj 4 --gpus $gpus --data_type $data_type --data ${data}
+fi
+
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+  echo "Score ..."
+  local/score.sh \
+    --stage 1 --stop-stage 2 \
+    --exp_dir $exp_dir \
+    --data ${data} \
+    --trials "$trials"
+fi
+
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  echo "Score norm ..."
+  local/score_norm.sh \
+    --stage 1 --stop-stage 3 \
+    --score_norm_method $score_norm_method \
+    --cohort_set cnceleb_train \
+    --top_n $top_n \
+    --exp_dir $exp_dir \
+    --data ${data} \
+    --trials "$trials"
+fi
+
+if [ ${stage} -le 7 ] && [ ${stop_stage} -ge 7 ]; then
+  echo "Export the best model ..."
+  python wespeaker/bin/export_jit.py \
+    --config $exp_dir/config.yaml \
+    --checkpoint $exp_dir/models/avg_model.pt \
+    --output_file $exp_dir/models/final.zip
+fi
+
+# ================== Large margin fine-tuning ==================
+# for reference: https://arxiv.org/abs/2206.11699
+# It shoule be noted that the large margin fine-tuning
+# is optional. It often be used in speaker verification
+# challenge to further improve performance. This training
+# proces will take longer segment as input and will take
+# up more gpu memory.
+
+if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
+  echo "Large margin fine-tuning ..."
+  lm_exp_dir=${exp_dir}-LM
+  mkdir -p ${lm_exp_dir}/models
+  # Use the pre-trained average model to initialize the LM training
+  cp ${exp_dir}/models/avg_model.pt ${lm_exp_dir}/models/model_0.pt
+  bash run.sh --stage 3 --stop_stage 7 \
+      --data ${data} \
+      --data_type ${data_type} \
+      --config ${lm_config} \
+      --exp_dir ${lm_exp_dir} \
+      --gpus $gpus \
+      --num_avg 1 \
+      --model_init "" \
+      --checkpoint ${lm_exp_dir}/models/model_0.pt \
+      --trials "$trials" \
+      --score_norm_method ${score_norm_method} \
+      --top_n ${top_n}
+fi
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/README.md b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..54dd9fae5b2019f47e24827b3260354c58f6a00e
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/README.md
@@ -0,0 +1,20 @@
+## Results for SRE16
+
+* Setup: fbank40, num_frms200, epoch150, Softmax, aug_prob0.6
+* Scoring: cosine & PLDA & PLDA Adaptation
+* Metric: EER(%)
+
+| Model                | Params | FLOPs  |  Backend   | Pooled | Tagalog | Cantonese |
+|:---------------------|:------:|:------:|:----------:|:------:|:-------:|:---------:|
+| ResNet34-TSTP-emb256 | 6.63M  | 4.55G  |   Cosine   |  15.4  |  19.82  |   10.39   |
+|                      |        |        |    PLDA    | 11.689 | 16.961  |   6.239   |
+|                      |        |        | Adapt PLDA | 5.788  |  8.974  |   2.674   |
+
+Current PLDA implementation is fully compatible with the Kaldi version, note that
+we can definitely improve the results with out adaptation with parameter tuning and extra LDA as shown in the Kaldi
+Recipe, we didn't do this because we focus more on the adapted results, which are good enough under current setup.
+
+* 🔥 UPDATE 2023.07.18: Support kaldi-compatible two-covariance PLDA and unsupervised domain adaptation.
+* 🔥 UPDATE 2023.07.14: Support
+  the [NIST SRE16 recipe](https://www.nist.gov/itl/iad/mig/speaker-recognition-evaluation-2016),
+  see [#177](https://github.com/wenet-e2e/wespeaker/pull/177).
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/conf/resnet.yaml b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/conf/resnet.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b50f4ce29596e76af88e381004c6b5d94da87faa
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/conf/resnet.yaml
@@ -0,0 +1,81 @@
+### train configuration
+
+exp_dir: exp/ResNet34-TSTP-emb256-fbank40-num_frms200-aug0.6-spFalse-saFalse-Softmax-SGD-epoch150
+gpus: "[0,1]"
+num_avg: 10
+enable_amp: False # whether enable automatic mixed precision training
+
+seed: 42
+num_epochs: 150
+save_epoch_interval: 5 # save model every 5 epochs
+log_batch_interval: 100 # log every 100 batchs
+
+dataloader_args:
+  batch_size: 256
+  num_workers: 16
+  pin_memory: False
+  prefetch_factor: 8
+  drop_last: True
+
+dataset_args:
+  # the sample number which will be traversed within one epoch, if the value equals to 0,
+  # the utterance number in the dataset will be used as the sample_num_per_epoch.
+  sample_num_per_epoch: 780000
+  shuffle: True
+  shuffle_args:
+    shuffle_size: 1500
+  filter: True
+  filter_args:
+    min_num_frames: 100
+    max_num_frames: 300
+  resample_rate: 8000
+  speed_perturb: False
+  num_frms: 200
+  aug_prob: 0.6 # prob to add reverb & noise aug per sample
+  fbank_args:
+    num_mel_bins: 40
+    frame_shift: 10
+    frame_length: 25
+    dither: 1.0
+  spec_aug: False
+  spec_aug_args:
+    num_t_mask: 1
+    num_f_mask: 1
+    max_t: 10
+    max_f: 8
+    prob: 0.6
+
+model: ResNet34 # ResNet18, ResNet34, ResNet50, ResNet101, ResNet152
+model_init: null
+model_args:
+  feat_dim: 40
+  embed_dim: 256
+  pooling_func: "TSTP" # TSTP, ASTP, MQMHASTP
+  two_emb_layer: False
+projection_args:
+  project_type: "softmax" # add_margin, arc_margin, sphere, softmax, arc_margin_intertopk_subcenter
+
+margin_scheduler: MarginScheduler
+margin_update:
+  initial_margin: 0.0
+  final_margin: 0.2
+  increase_start_epoch: 20
+  fix_start_epoch: 40
+  update_margin: True
+  increase_type: "exp" # exp, linear
+
+loss: CrossEntropyLoss
+loss_args: {}
+
+optimizer: SGD
+optimizer_args:
+  momentum: 0.9
+  nesterov: True
+  weight_decay: 0.0001
+
+scheduler: ExponentialDecrease
+scheduler_args:
+  initial_lr: 0.1
+  final_lr: 0.00005
+  warm_up_epoch: 6
+  warm_from_zero: True
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/extract_sre.sh b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/extract_sre.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c36ce5dbb9fa1ae59c018383e5116d6187c3ec4f
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/extract_sre.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Copyright (c) 2022 Hongji Wang (jijijiang77@gmail.com)
+#               2023 Zhengyang Chen (chenzhengyang117@gmail.com)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exp_dir=''
+model_path=''
+nj=4
+gpus="[0,1]"
+data_type="shard"  # shard/raw/feat
+data=data
+reverb_data=data/rirs/lmdb
+noise_data=data/musan/lmdb
+aug_plda_data=0
+
+. tools/parse_options.sh
+set -e
+
+if [ $aug_plda_data = 0 ];then
+    sre_plda_data=sre
+else
+    sre_plda_data=sre_aug
+fi
+
+data_name_array=(
+    "${sre_plda_data}"
+    "sre16_major"
+    "sre16_eval_enroll"
+    "sre16_eval_test"
+)
+data_list_path_array=(
+    "${data}/${sre_plda_data}/${data_type}.list"
+    "${data}/sre16_major/${data_type}.list"
+    "${data}/sre16_eval_enroll/${data_type}.list"
+    "${data}/sre16_eval_test/${data_type}.list"
+)
+data_scp_path_array=(
+    "${data}/${sre_plda_data}/wav.scp"
+    "${data}/sre16_major/wav.scp"
+    "${data}/sre16_eval_enroll/wav.scp"
+    "${data}/sre16_eval_test/wav.scp"
+) # to count the number of wavs
+nj_array=($nj $nj $nj $nj)
+batch_size_array=(1 1 1 1) # batch_size of test set must be 1 !!!
+num_workers_array=(1 1 1 1)
+if [ $aug_plda_data = 0 ];then
+    aug_prob_array=(0.0 0.0 0.0 0.0)
+else
+    aug_prob_array=(0.67 0.0 0.0 0.0)
+fi
+count=${#data_name_array[@]}
+
+for i in $(seq 0 $(($count - 1))); do
+  wavs_num=$(wc -l ${data_scp_path_array[$i]} | awk '{print $1}')
+  bash tools/extract_embedding.sh --exp_dir ${exp_dir} \
+    --model_path $model_path \
+    --data_type ${data_type} \
+    --data_list ${data_list_path_array[$i]} \
+    --wavs_num ${wavs_num} \
+    --store_dir ${data_name_array[$i]} \
+    --batch_size ${batch_size_array[$i]} \
+    --num_workers ${num_workers_array[$i]} \
+    --aug_prob ${aug_prob_array[$i]} \
+    --reverb_data ${reverb_data} \
+    --noise_data ${noise_data} \
+    --nj ${nj_array[$i]} \
+    --gpus $gpus
+done
+
+wait
+
+echo "mean vector of enroll"
+python tools/vector_mean.py \
+  --spk2utt ${data}/sre16_eval_enroll/spk2utt \
+  --xvector_scp $exp_dir/embeddings/sre16_eval_enroll/xvector.scp \
+  --spk_xvector_ark $exp_dir/embeddings/sre16_eval_enroll/enroll_spk_xvector.ark
+
+mkdir -p ${exp_dir}/embeddings/eval
+cat ${exp_dir}/embeddings/sre16_eval_enroll/enroll_spk_xvector.scp \
+    ${exp_dir}/embeddings/sre16_eval_test/xvector.scp \
+    > ${exp_dir}/embeddings/eval/xvector.scp
+
+echo "Embedding dir is (${exp_dir}/embeddings)."
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/filter_utt_accd_dur.py b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/filter_utt_accd_dur.py
new file mode 100644
index 0000000000000000000000000000000000000000..fff80d970af290e310f27fdb4903009cb7a1331d
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/filter_utt_accd_dur.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2023 Zhengyang Chen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fire
+
+
+def main(wav_scp, utt2voice_dur, filter_wav_scp, dur_thres=5.0):
+
+    utt2voice_dur_dict = {}
+    with open(utt2voice_dur, "r") as f:
+        for line in f:
+            utt, dur = line.strip().split()
+            utt2voice_dur_dict[utt] = float(dur)
+
+    with open(wav_scp, "r") as f, open(filter_wav_scp, "w") as fw:
+        for line in f:
+            utt = line.strip().split()[0]
+            if utt in utt2voice_dur_dict:
+                if utt2voice_dur_dict[utt] > dur_thres:
+                    fw.write(line)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/generate_sre_aug.py b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/generate_sre_aug.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5fdfb14f3e65fba2c9d61d6a680f9451d93f0c4
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/generate_sre_aug.py
@@ -0,0 +1,56 @@
+# Copyright (c) 2023 Zhengyang Chen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import fire
+
+
+def main(ori_dir, aug_dir, aug_copy_num=2):
+
+    if not os.path.exists(aug_dir):
+        os.makedirs(aug_dir)
+
+    read_wav_scp = os.path.join(ori_dir, 'wav.scp')
+    aug_wav_scp = os.path.join(aug_dir, 'wav.scp')
+    read_utt2spk = os.path.join(ori_dir, 'utt2spk')
+    aug_utt2spk = os.path.join(aug_dir, 'utt2spk')
+    read_vad = os.path.join(ori_dir, 'vad')
+    store_vad = os.path.join(aug_dir, 'vad')
+
+    with open(read_wav_scp, 'r') as f, open(aug_wav_scp, 'w') as wf:
+        for line in f:
+            line = line.strip().split()
+            utt, other_info = line[0], ' '.join(line[1:])
+            for i in range(aug_copy_num + 1):
+                wf.write(utt + '_copy-' + str(i) + ' ' + other_info + '\n')
+
+    with open(read_utt2spk, 'r') as f, open(aug_utt2spk, 'w') as wf:
+        for line in f:
+            line = line.strip().split()
+            utt, spk = line[0], line[1]
+            for i in range(aug_copy_num + 1):
+                wf.write(utt + '_copy-' + str(i) + ' ' + spk + '\n')
+
+    with open(read_vad, 'r') as f, open(store_vad, 'w') as wf:
+        for line in f:
+            line = line.strip().split()
+            seg, utt, vad = line[0], line[1], ' '.join(line[2:])
+            for i in range(aug_copy_num + 1):
+                new_seg = seg + '_copy-' + str(i)
+                new_utt = utt + '_copy-' + str(i)
+                wf.write(new_seg + ' ' + new_utt + ' ' + vad + '\n')
+
+
+if __name__ == "__main__":
+    fire.Fire(main)
diff --git a/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/make_system_sad.py b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/make_system_sad.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c629ea404c5df594c25ad7f3bedc09684c6c7f0
--- /dev/null
+++ b/MOSS_Speaker-RoPE/wespeaker/examples/sre/v2/local/make_system_sad.py
@@ -0,0 +1,138 @@
+# Copyright (c) 2022 Xu Xiang
+#               2023 Zhengyang Chen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["OPENBLAS_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
+os.environ["NUMEXPR_NUM_THREADS"] = "1"
+
+import sys
+import io
+import functools
+import concurrent.futures
+import argparse
+import importlib
+import torchaudio
+import subprocess
+
+import torch
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='')
+    parser.add_argument('--repo-path',
+                        required=True,
+                        help='VAD model repo path')
+    parser.add_argument('--scp', required=True, help='wav scp')
+    parser.add_argument('--min-duration',
+                        required=True,
+                        type=float,
+                        help='min duration')
+    args = parser.parse_args()
+
+    return args
+
+
+@functools.lru_cache(maxsize=1)
+def load_wav(wav_rxfilename, ):
+    """ This function reads audio file and return data in pytorch tensor.
+        "lru_cache" holds recently loaded audio so that can be called
+        many times on the same audio file.
+        OPTIMIZE: controls lru_cache size for random access,
+        considering memory size
+    """
+    if wav_rxfilename.endswith('|'):
+        # input piped command
+        p = subprocess.Popen(wav_rxfilename[:-1],
+                             shell=True,
+                             stdout=subprocess.PIPE)
+        data, samplerate = torchaudio.load(io.BytesIO(p.stdout.read()))
+    elif wav_rxfilename == '-':
+        # stdin
+        data, samplerate = torchaudio.load(sys.stdin)
+    else:
+        # normal wav file
+        data, samplerate = torchaudio.load(wav_rxfilename)
+    return data.squeeze(0), samplerate
+
+
+def read_scp(scp):
+    utt_wav_pair = []
+    for line in open(scp, 'r'):
+        segs = line.strip().split()
+        if len(segs) > 2:
+            utt, wav = segs[0], ' '.join(segs[1:])
+        else:
+            utt, wav = segs[0], segs[1]
+        utt_wav_pair.append((utt, wav))
+
+    return utt_wav_pair
+
+
+def silero_vad(utt_wav_pair,
+               repo_path,
+               min_duration,
+               sampling_rate=8000,
+               threshold=0.25):
+
+    def module_from_file(module_name, file_path):
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        return module
+
+    utils_vad = module_from_file("utils_vad",
+                                 os.path.join(repo_path, "utils_vad.py"))
+    model = utils_vad.init_jit_model(
+        os.path.join(repo_path, 'files/silero_vad.jit'))
+
+    utt, wav = utt_wav_pair
+
+    wav, sr = load_wav(wav)
+    assert sr == sampling_rate
+    speech_timestamps = utils_vad.get_speech_timestamps(
+        wav, model, sampling_rate=sampling_rate, threshold=threshold)
+
+    vad_result = ""
+    for item in speech_timestamps:
+        begin = item['start'] / sampling_rate
+        end = item['end'] / sampling_rate
+        if end - begin >= min_duration:
+            vad_result += "{}-{:08d}-{:08d} {} {:.3f} {:.3f}\n".format(
+                utt, int(begin * 1000), int(end * 1000), utt, begin, end)
+
+    return vad_result
+
+
+def main():
+    args = get_args()
+
+    vad = functools.partial(silero_vad,
+                            repo_path=args.repo_path,
+                            min_duration=args.min_duration)
+    utt_wav_pair_list = read_scp(args.scp)
+
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        print(''.join(executor.map(vad, utt_wav_pair_list)), end='')
+
+
+if __name__ == '__main__':
+    torch.set_num_threads(1)
+
+    main()