Upload folder using huggingface_hub

12_128_e3_3e-5/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

12_128_e3_3e-5/README.md

@@ -0,0 +1,63 @@
+---
+library_name: peft
+license: apache-2.0
+base_model: ibm-granite/granite-3.3-8b-base
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- data/knowledge_lora_training_data_1000
+model-index:
+- name: 12_128_e3_3e-5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 12_128_e3_3e-5
+
+This model is a fine-tuned version of [ibm-granite/granite-3.3-8b-base](https://huggingface.co/ibm-granite/granite-3.3-8b-base) on the data/knowledge_lora_training_data_1000 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 64
+- optimizer: Use adamw_torch with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.15.2
+- Transformers 4.52.4
+- Pytorch 2.7.0+cu126
+- Datasets 3.6.0
+- Tokenizers 0.21.2

12_128_e3_3e-5/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "ibm-granite/granite-3.3-8b-base",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj",
+    "down_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

12_128_e3_3e-5/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2192d8ad90a9aaaac4b6b47909103b1b0ecc886a9ac386a7ecc00975a9592425
+size 791751704

12_128_e3_3e-5/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.619647455794561e+18,
+    "train_loss": 0.537263454939868,
+    "train_runtime": 774.41,
+    "train_samples": 11817,
+    "train_samples_per_second": 45.778,
+    "train_steps_per_second": 1.433
+}

12_128_e3_3e-5/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "GraniteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0,
+  "attention_multiplier": 0.0078125,
+  "bos_token_id": 0,
+  "embedding_multiplier": 12.0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 12800,
+  "logits_scaling": 16.0,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "granite",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "residual_multiplier": 0.22,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 49152
+}

12_128_e3_3e-5/special_tokens_map.json

@@ -0,0 +1,45 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<reponame>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<reponame>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

12_128_e3_3e-5/tokenizer_config.json

@@ -0,0 +1,188 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<commit_before>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<commit_msg>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<commit_after>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<reponame>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<reponame>",
+  "padding_side": "left",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

12_128_e3_3e-5/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.619647455794561e+18,
+    "train_loss": 0.537263454939868,
+    "train_runtime": 774.41,
+    "train_samples": 11817,
+    "train_samples_per_second": 45.778,
+    "train_steps_per_second": 1.433
+}

12_128_e3_3e-5/trainer_state.json

@@ -0,0 +1,1597 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1110,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013531799729364006,
+      "grad_norm": 1.4069883823394775,
+      "learning_rate": 2.1428571428571427e-06,
+      "loss": 1.3069,
+      "step": 5
+    },
+    {
+      "epoch": 0.02706359945872801,
+      "grad_norm": 0.7561248540878296,
+      "learning_rate": 4.821428571428572e-06,
+      "loss": 1.3133,
+      "step": 10
+    },
+    {
+      "epoch": 0.04059539918809202,
+      "grad_norm": 0.601601779460907,
+      "learning_rate": 7.5e-06,
+      "loss": 1.296,
+      "step": 15
+    },
+    {
+      "epoch": 0.05412719891745602,
+      "grad_norm": 0.691739022731781,
+      "learning_rate": 1.0178571428571429e-05,
+      "loss": 1.2094,
+      "step": 20
+    },
+    {
+      "epoch": 0.06765899864682003,
+      "grad_norm": 0.5830556750297546,
+      "learning_rate": 1.2857142857142857e-05,
+      "loss": 1.2479,
+      "step": 25
+    },
+    {
+      "epoch": 0.08119079837618404,
+      "grad_norm": 0.5143088102340698,
+      "learning_rate": 1.553571428571429e-05,
+      "loss": 1.2389,
+      "step": 30
+    },
+    {
+      "epoch": 0.09472259810554803,
+      "grad_norm": 0.45757943391799927,
+      "learning_rate": 1.8214285714285712e-05,
+      "loss": 1.1958,
+      "step": 35
+    },
+    {
+      "epoch": 0.10825439783491204,
+      "grad_norm": 0.501616895198822,
+      "learning_rate": 2.089285714285714e-05,
+      "loss": 1.17,
+      "step": 40
+    },
+    {
+      "epoch": 0.12178619756427606,
+      "grad_norm": 0.45063820481300354,
+      "learning_rate": 2.357142857142857e-05,
+      "loss": 1.1523,
+      "step": 45
+    },
+    {
+      "epoch": 0.13531799729364005,
+      "grad_norm": 0.4993128180503845,
+      "learning_rate": 2.625e-05,
+      "loss": 1.1673,
+      "step": 50
+    },
+    {
+      "epoch": 0.14884979702300405,
+      "grad_norm": 0.43754082918167114,
+      "learning_rate": 2.892857142857143e-05,
+      "loss": 1.1412,
+      "step": 55
+    },
+    {
+      "epoch": 0.16238159675236807,
+      "grad_norm": 0.5136508941650391,
+      "learning_rate": 2.999940032022394e-05,
+      "loss": 1.1245,
+      "step": 60
+    },
+    {
+      "epoch": 0.17591339648173207,
+      "grad_norm": 0.49709609150886536,
+      "learning_rate": 2.9995735784121558e-05,
+      "loss": 1.1651,
+      "step": 65
+    },
+    {
+      "epoch": 0.18944519621109607,
+      "grad_norm": 0.42540761828422546,
+      "learning_rate": 2.9988740680258434e-05,
+      "loss": 1.0692,
+      "step": 70
+    },
+    {
+      "epoch": 0.2029769959404601,
+      "grad_norm": 0.49962568283081055,
+      "learning_rate": 2.9978416562254055e-05,
+      "loss": 1.0704,
+      "step": 75
+    },
+    {
+      "epoch": 0.2165087956698241,
+      "grad_norm": 0.5214064121246338,
+      "learning_rate": 2.996476572310524e-05,
+      "loss": 1.1607,
+      "step": 80
+    },
+    {
+      "epoch": 0.23004059539918809,
+      "grad_norm": 0.47680965065956116,
+      "learning_rate": 2.9947791194676896e-05,
+      "loss": 1.0984,
+      "step": 85
+    },
+    {
+      "epoch": 0.2435723951285521,
+      "grad_norm": 0.5332472324371338,
+      "learning_rate": 2.9927496747028565e-05,
+      "loss": 1.0906,
+      "step": 90
+    },
+    {
+      "epoch": 0.2571041948579161,
+      "grad_norm": 0.5810690522193909,
+      "learning_rate": 2.9903886887577177e-05,
+      "loss": 1.0073,
+      "step": 95
+    },
+    {
+      "epoch": 0.2706359945872801,
+      "grad_norm": 0.4904935657978058,
+      "learning_rate": 2.987696686009588e-05,
+      "loss": 1.0417,
+      "step": 100
+    },
+    {
+      "epoch": 0.28416779431664413,
+      "grad_norm": 0.5119685530662537,
+      "learning_rate": 2.9846742643549418e-05,
+      "loss": 1.0239,
+      "step": 105
+    },
+    {
+      "epoch": 0.2976995940460081,
+      "grad_norm": 0.6267646551132202,
+      "learning_rate": 2.9813220950766205e-05,
+      "loss": 1.0139,
+      "step": 110
+    },
+    {
+      "epoch": 0.3112313937753721,
+      "grad_norm": 0.698165237903595,
+      "learning_rate": 2.9776409226947394e-05,
+      "loss": 1.0486,
+      "step": 115
+    },
+    {
+      "epoch": 0.32476319350473615,
+      "grad_norm": 0.6429921388626099,
+      "learning_rate": 2.9736315648013272e-05,
+      "loss": 0.9811,
+      "step": 120
+    },
+    {
+      "epoch": 0.3382949932341001,
+      "grad_norm": 0.6176176071166992,
+      "learning_rate": 2.969294911878742e-05,
+      "loss": 0.9885,
+      "step": 125
+    },
+    {
+      "epoch": 0.35182679296346414,
+      "grad_norm": 0.6764014959335327,
+      "learning_rate": 2.9646319271018888e-05,
+      "loss": 0.9593,
+      "step": 130
+    },
+    {
+      "epoch": 0.36535859269282817,
+      "grad_norm": 0.5621255040168762,
+      "learning_rate": 2.959643646124303e-05,
+      "loss": 0.9664,
+      "step": 135
+    },
+    {
+      "epoch": 0.37889039242219213,
+      "grad_norm": 0.7597676515579224,
+      "learning_rate": 2.9543311768481276e-05,
+      "loss": 0.9983,
+      "step": 140
+    },
+    {
+      "epoch": 0.39242219215155616,
+      "grad_norm": 0.7237449884414673,
+      "learning_rate": 2.9486956991780463e-05,
+      "loss": 0.8973,
+      "step": 145
+    },
+    {
+      "epoch": 0.4059539918809202,
+      "grad_norm": 0.6780668497085571,
+      "learning_rate": 2.942738464759229e-05,
+      "loss": 0.929,
+      "step": 150
+    },
+    {
+      "epoch": 0.41948579161028415,
+      "grad_norm": 0.7538473606109619,
+      "learning_rate": 2.9364607966993347e-05,
+      "loss": 0.9377,
+      "step": 155
+    },
+    {
+      "epoch": 0.4330175913396482,
+      "grad_norm": 0.634324312210083,
+      "learning_rate": 2.929864089274655e-05,
+      "loss": 0.928,
+      "step": 160
+    },
+    {
+      "epoch": 0.4465493910690122,
+      "grad_norm": 0.6696556806564331,
+      "learning_rate": 2.922949807620438e-05,
+      "loss": 0.9463,
+      "step": 165
+    },
+    {
+      "epoch": 0.46008119079837617,
+      "grad_norm": 0.697542130947113,
+      "learning_rate": 2.9157194874054826e-05,
+      "loss": 0.9139,
+      "step": 170
+    },
+    {
+      "epoch": 0.4736129905277402,
+      "grad_norm": 0.649330198764801,
+      "learning_rate": 2.908174734491066e-05,
+      "loss": 0.8718,
+      "step": 175
+    },
+    {
+      "epoch": 0.4871447902571042,
+      "grad_norm": 0.6946331262588501,
+      "learning_rate": 2.9003172245742778e-05,
+      "loss": 0.8492,
+      "step": 180
+    },
+    {
+      "epoch": 0.5006765899864682,
+      "grad_norm": 0.7980474829673767,
+      "learning_rate": 2.892148702815848e-05,
+      "loss": 0.8853,
+      "step": 185
+    },
+    {
+      "epoch": 0.5142083897158322,
+      "grad_norm": 0.7989095449447632,
+      "learning_rate": 2.8836709834525455e-05,
+      "loss": 0.8464,
+      "step": 190
+    },
+    {
+      "epoch": 0.5277401894451962,
+      "grad_norm": 0.7314903140068054,
+      "learning_rate": 2.874885949394231e-05,
+      "loss": 0.8859,
+      "step": 195
+    },
+    {
+      "epoch": 0.5412719891745602,
+      "grad_norm": 0.7945508360862732,
+      "learning_rate": 2.8657955518056653e-05,
+      "loss": 0.8938,
+      "step": 200
+    },
+    {
+      "epoch": 0.5548037889039242,
+      "grad_norm": 0.8844264149665833,
+      "learning_rate": 2.8564018096731507e-05,
+      "loss": 0.8111,
+      "step": 205
+    },
+    {
+      "epoch": 0.5683355886332883,
+      "grad_norm": 0.850426197052002,
+      "learning_rate": 2.8467068093561125e-05,
+      "loss": 0.813,
+      "step": 210
+    },
+    {
+      "epoch": 0.5818673883626523,
+      "grad_norm": 0.7970026135444641,
+      "learning_rate": 2.8367127041237178e-05,
+      "loss": 0.8473,
+      "step": 215
+    },
+    {
+      "epoch": 0.5953991880920162,
+      "grad_norm": 0.879941999912262,
+      "learning_rate": 2.8264217136766306e-05,
+      "loss": 0.7922,
+      "step": 220
+    },
+    {
+      "epoch": 0.6089309878213802,
+      "grad_norm": 0.8317419290542603,
+      "learning_rate": 2.8158361236540166e-05,
+      "loss": 0.8137,
+      "step": 225
+    },
+    {
+      "epoch": 0.6224627875507442,
+      "grad_norm": 0.8715953230857849,
+      "learning_rate": 2.8049582851258958e-05,
+      "loss": 0.8367,
+      "step": 230
+    },
+    {
+      "epoch": 0.6359945872801083,
+      "grad_norm": 0.7387327551841736,
+      "learning_rate": 2.7937906140709742e-05,
+      "loss": 0.8231,
+      "step": 235
+    },
+    {
+      "epoch": 0.6495263870094723,
+      "grad_norm": 0.8666847944259644,
+      "learning_rate": 2.782335590840046e-05,
+      "loss": 0.8073,
+      "step": 240
+    },
+    {
+      "epoch": 0.6630581867388363,
+      "grad_norm": 0.7476521730422974,
+      "learning_rate": 2.7705957596051096e-05,
+      "loss": 0.7582,
+      "step": 245
+    },
+    {
+      "epoch": 0.6765899864682002,
+      "grad_norm": 0.9227946400642395,
+      "learning_rate": 2.7585737277943023e-05,
+      "loss": 0.7905,
+      "step": 250
+    },
+    {
+      "epoch": 0.6901217861975643,
+      "grad_norm": 1.0331108570098877,
+      "learning_rate": 2.7462721655127897e-05,
+      "loss": 0.7567,
+      "step": 255
+    },
+    {
+      "epoch": 0.7036535859269283,
+      "grad_norm": 0.8921414613723755,
+      "learning_rate": 2.73369380494973e-05,
+      "loss": 0.7819,
+      "step": 260
+    },
+    {
+      "epoch": 0.7171853856562923,
+      "grad_norm": 0.7794017195701599,
+      "learning_rate": 2.7208414397714553e-05,
+      "loss": 0.7567,
+      "step": 265
+    },
+    {
+      "epoch": 0.7307171853856563,
+      "grad_norm": 0.9170422554016113,
+      "learning_rate": 2.7077179245009923e-05,
+      "loss": 0.7738,
+      "step": 270
+    },
+    {
+      "epoch": 0.7442489851150202,
+      "grad_norm": 0.9383819699287415,
+      "learning_rate": 2.6943261738840738e-05,
+      "loss": 0.7317,
+      "step": 275
+    },
+    {
+      "epoch": 0.7577807848443843,
+      "grad_norm": 0.8380761742591858,
+      "learning_rate": 2.6806691622417662e-05,
+      "loss": 0.7316,
+      "step": 280
+    },
+    {
+      "epoch": 0.7713125845737483,
+      "grad_norm": 0.9718127250671387,
+      "learning_rate": 2.6667499228098735e-05,
+      "loss": 0.6751,
+      "step": 285
+    },
+    {
+      "epoch": 0.7848443843031123,
+      "grad_norm": 0.8734492063522339,
+      "learning_rate": 2.6525715470652516e-05,
+      "loss": 0.7215,
+      "step": 290
+    },
+    {
+      "epoch": 0.7983761840324763,
+      "grad_norm": 0.9708399772644043,
+      "learning_rate": 2.6381371840391862e-05,
+      "loss": 0.7302,
+      "step": 295
+    },
+    {
+      "epoch": 0.8119079837618404,
+      "grad_norm": 0.8662261962890625,
+      "learning_rate": 2.6234500396179943e-05,
+      "loss": 0.6971,
+      "step": 300
+    },
+    {
+      "epoch": 0.8254397834912043,
+      "grad_norm": 0.8830039501190186,
+      "learning_rate": 2.6085133758309887e-05,
+      "loss": 0.6686,
+      "step": 305
+    },
+    {
+      "epoch": 0.8389715832205683,
+      "grad_norm": 0.9637537598609924,
+      "learning_rate": 2.5933305101259812e-05,
+      "loss": 0.6769,
+      "step": 310
+    },
+    {
+      "epoch": 0.8525033829499323,
+      "grad_norm": 1.025269627571106,
+      "learning_rate": 2.5779048146324735e-05,
+      "loss": 0.6829,
+      "step": 315
+    },
+    {
+      "epoch": 0.8660351826792964,
+      "grad_norm": 0.9953080415725708,
+      "learning_rate": 2.562239715412703e-05,
+      "loss": 0.6753,
+      "step": 320
+    },
+    {
+      "epoch": 0.8795669824086604,
+      "grad_norm": 0.8558081984519958,
+      "learning_rate": 2.5463386917007118e-05,
+      "loss": 0.6315,
+      "step": 325
+    },
+    {
+      "epoch": 0.8930987821380244,
+      "grad_norm": 0.9724118709564209,
+      "learning_rate": 2.5302052751296064e-05,
+      "loss": 0.695,
+      "step": 330
+    },
+    {
+      "epoch": 0.9066305818673883,
+      "grad_norm": 0.8320069313049316,
+      "learning_rate": 2.5138430489471753e-05,
+      "loss": 0.6803,
+      "step": 335
+    },
+    {
+      "epoch": 0.9201623815967523,
+      "grad_norm": 0.9561787843704224,
+      "learning_rate": 2.4972556472200507e-05,
+      "loss": 0.6344,
+      "step": 340
+    },
+    {
+      "epoch": 0.9336941813261164,
+      "grad_norm": 0.8946269750595093,
+      "learning_rate": 2.4804467540265734e-05,
+      "loss": 0.6594,
+      "step": 345
+    },
+    {
+      "epoch": 0.9472259810554804,
+      "grad_norm": 0.9975743889808655,
+      "learning_rate": 2.46342010263856e-05,
+      "loss": 0.6589,
+      "step": 350
+    },
+    {
+      "epoch": 0.9607577807848444,
+      "grad_norm": 0.9772000312805176,
+      "learning_rate": 2.4461794746921375e-05,
+      "loss": 0.648,
+      "step": 355
+    },
+    {
+      "epoch": 0.9742895805142084,
+      "grad_norm": 0.9670999050140381,
+      "learning_rate": 2.4287286993478373e-05,
+      "loss": 0.6745,
+      "step": 360
+    },
+    {
+      "epoch": 0.9878213802435724,
+      "grad_norm": 1.0077227354049683,
+      "learning_rate": 2.4110716524401367e-05,
+      "loss": 0.6473,
+      "step": 365
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.554697871208191,
+      "learning_rate": 2.3932122556166317e-05,
+      "loss": 0.6547,
+      "step": 370
+    },
+    {
+      "epoch": 1.013531799729364,
+      "grad_norm": 1.3504902124404907,
+      "learning_rate": 2.375154475467034e-05,
+      "loss": 0.5261,
+      "step": 375
+    },
+    {
+      "epoch": 1.027063599458728,
+      "grad_norm": 1.0020747184753418,
+      "learning_rate": 2.3569023226421885e-05,
+      "loss": 0.536,
+      "step": 380
+    },
+    {
+      "epoch": 1.040595399188092,
+      "grad_norm": 1.35636305809021,
+      "learning_rate": 2.3384598509633044e-05,
+      "loss": 0.5492,
+      "step": 385
+    },
+    {
+      "epoch": 1.054127198917456,
+      "grad_norm": 1.0515373945236206,
+      "learning_rate": 2.3198311565215963e-05,
+      "loss": 0.5478,
+      "step": 390
+    },
+    {
+      "epoch": 1.0676589986468201,
+      "grad_norm": 0.9901190400123596,
+      "learning_rate": 2.3010203767685384e-05,
+      "loss": 0.5278,
+      "step": 395
+    },
+    {
+      "epoch": 1.0811907983761841,
+      "grad_norm": 0.9804115295410156,
+      "learning_rate": 2.2820316895969337e-05,
+      "loss": 0.5302,
+      "step": 400
+    },
+    {
+      "epoch": 1.094722598105548,
+      "grad_norm": 0.8747122287750244,
+      "learning_rate": 2.262869312412998e-05,
+      "loss": 0.5197,
+      "step": 405
+    },
+    {
+      "epoch": 1.108254397834912,
+      "grad_norm": 0.9555748701095581,
+      "learning_rate": 2.2435375011996718e-05,
+      "loss": 0.5836,
+      "step": 410
+    },
+    {
+      "epoch": 1.121786197564276,
+      "grad_norm": 1.0065706968307495,
+      "learning_rate": 2.2240405495713596e-05,
+      "loss": 0.5212,
+      "step": 415
+    },
+    {
+      "epoch": 1.13531799729364,
+      "grad_norm": 0.9913938641548157,
+      "learning_rate": 2.2043827878203168e-05,
+      "loss": 0.5102,
+      "step": 420
+    },
+    {
+      "epoch": 1.148849797023004,
+      "grad_norm": 1.1417410373687744,
+      "learning_rate": 2.184568581954887e-05,
+      "loss": 0.5005,
+      "step": 425
+    },
+    {
+      "epoch": 1.162381596752368,
+      "grad_norm": 1.0159755945205688,
+      "learning_rate": 2.1646023327298046e-05,
+      "loss": 0.5643,
+      "step": 430
+    },
+    {
+      "epoch": 1.175913396481732,
+      "grad_norm": 0.9843266606330872,
+      "learning_rate": 2.144488474668785e-05,
+      "loss": 0.4968,
+      "step": 435
+    },
+    {
+      "epoch": 1.1894451962110961,
+      "grad_norm": 1.197916865348816,
+      "learning_rate": 2.1242314750796133e-05,
+      "loss": 0.4889,
+      "step": 440
+    },
+    {
+      "epoch": 1.2029769959404601,
+      "grad_norm": 0.9884390830993652,
+      "learning_rate": 2.1038358330619494e-05,
+      "loss": 0.4976,
+      "step": 445
+    },
+    {
+      "epoch": 1.2165087956698242,
+      "grad_norm": 1.1356792449951172,
+      "learning_rate": 2.0833060785080733e-05,
+      "loss": 0.5087,
+      "step": 450
+    },
+    {
+      "epoch": 1.230040595399188,
+      "grad_norm": 1.1139668226242065,
+      "learning_rate": 2.0626467710967915e-05,
+      "loss": 0.5075,
+      "step": 455
+    },
+    {
+      "epoch": 1.243572395128552,
+      "grad_norm": 0.992261528968811,
+      "learning_rate": 2.0418624992807297e-05,
+      "loss": 0.5029,
+      "step": 460
+    },
+    {
+      "epoch": 1.257104194857916,
+      "grad_norm": 1.295310616493225,
+      "learning_rate": 2.0209578792672304e-05,
+      "loss": 0.5218,
+      "step": 465
+    },
+    {
+      "epoch": 1.27063599458728,
+      "grad_norm": 1.0917266607284546,
+      "learning_rate": 1.9999375539930915e-05,
+      "loss": 0.4967,
+      "step": 470
+    },
+    {
+      "epoch": 1.284167794316644,
+      "grad_norm": 1.1314043998718262,
+      "learning_rate": 1.9788061920933612e-05,
+      "loss": 0.4815,
+      "step": 475
+    },
+    {
+      "epoch": 1.297699594046008,
+      "grad_norm": 0.9803086519241333,
+      "learning_rate": 1.9575684868644352e-05,
+      "loss": 0.4589,
+      "step": 480
+    },
+    {
+      "epoch": 1.3112313937753721,
+      "grad_norm": 1.0493569374084473,
+      "learning_rate": 1.9362291552216657e-05,
+      "loss": 0.5341,
+      "step": 485
+    },
+    {
+      "epoch": 1.3247631935047361,
+      "grad_norm": 1.5354151725769043,
+      "learning_rate": 1.9147929366517325e-05,
+      "loss": 0.5093,
+      "step": 490
+    },
+    {
+      "epoch": 1.3382949932341002,
+      "grad_norm": 1.098484992980957,
+      "learning_rate": 1.893264592159998e-05,
+      "loss": 0.5219,
+      "step": 495
+    },
+    {
+      "epoch": 1.3518267929634642,
+      "grad_norm": 1.033700942993164,
+      "learning_rate": 1.871648903213084e-05,
+      "loss": 0.415,
+      "step": 500
+    },
+    {
+      "epoch": 1.3653585926928282,
+      "grad_norm": 1.2974547147750854,
+      "learning_rate": 1.8499506706768995e-05,
+      "loss": 0.4666,
+      "step": 505
+    },
+    {
+      "epoch": 1.3788903924221922,
+      "grad_norm": 1.075243353843689,
+      "learning_rate": 1.8281747137503697e-05,
+      "loss": 0.419,
+      "step": 510
+    },
+    {
+      "epoch": 1.3924221921515563,
+      "grad_norm": 1.2342406511306763,
+      "learning_rate": 1.8063258688950818e-05,
+      "loss": 0.4551,
+      "step": 515
+    },
+    {
+      "epoch": 1.4059539918809203,
+      "grad_norm": 0.9819617867469788,
+      "learning_rate": 1.784408988761104e-05,
+      "loss": 0.4645,
+      "step": 520
+    },
+    {
+      "epoch": 1.419485791610284,
+      "grad_norm": 1.0715951919555664,
+      "learning_rate": 1.762428941109209e-05,
+      "loss": 0.4364,
+      "step": 525
+    },
+    {
+      "epoch": 1.4330175913396481,
+      "grad_norm": 1.0804096460342407,
+      "learning_rate": 1.7403906077297358e-05,
+      "loss": 0.445,
+      "step": 530
+    },
+    {
+      "epoch": 1.4465493910690121,
+      "grad_norm": 0.986536979675293,
+      "learning_rate": 1.718298883358342e-05,
+      "loss": 0.4235,
+      "step": 535
+    },
+    {
+      "epoch": 1.4600811907983762,
+      "grad_norm": 1.2482786178588867,
+      "learning_rate": 1.6961586745888772e-05,
+      "loss": 0.464,
+      "step": 540
+    },
+    {
+      "epoch": 1.4736129905277402,
+      "grad_norm": 1.055936574935913,
+      "learning_rate": 1.6739748987836247e-05,
+      "loss": 0.4386,
+      "step": 545
+    },
+    {
+      "epoch": 1.4871447902571042,
+      "grad_norm": 1.1653422117233276,
+      "learning_rate": 1.6517524829811485e-05,
+      "loss": 0.4202,
+      "step": 550
+    },
+    {
+      "epoch": 1.5006765899864682,
+      "grad_norm": 1.1142315864562988,
+      "learning_rate": 1.6294963628019942e-05,
+      "loss": 0.4352,
+      "step": 555
+    },
+    {
+      "epoch": 1.514208389715832,
+      "grad_norm": 1.1472088098526,
+      "learning_rate": 1.607211481352483e-05,
+      "loss": 0.4172,
+      "step": 560
+    },
+    {
+      "epoch": 1.527740189445196,
+      "grad_norm": 1.0435978174209595,
+      "learning_rate": 1.5849027881268434e-05,
+      "loss": 0.4175,
+      "step": 565
+    },
+    {
+      "epoch": 1.54127198917456,
+      "grad_norm": 1.2088128328323364,
+      "learning_rate": 1.5625752379079225e-05,
+      "loss": 0.4417,
+      "step": 570
+    },
+    {
+      "epoch": 1.5548037889039241,
+      "grad_norm": 1.206325888633728,
+      "learning_rate": 1.5402337896667243e-05,
+      "loss": 0.4372,
+      "step": 575
+    },
+    {
+      "epoch": 1.5683355886332881,
+      "grad_norm": 1.300421953201294,
+      "learning_rate": 1.5178834054610161e-05,
+      "loss": 0.4577,
+      "step": 580
+    },
+    {
+      "epoch": 1.5818673883626522,
+      "grad_norm": 1.2354118824005127,
+      "learning_rate": 1.4955290493332539e-05,
+      "loss": 0.4088,
+      "step": 585
+    },
+    {
+      "epoch": 1.5953991880920162,
+      "grad_norm": 1.083181619644165,
+      "learning_rate": 1.4731756862080606e-05,
+      "loss": 0.4597,
+      "step": 590
+    },
+    {
+      "epoch": 1.6089309878213802,
+      "grad_norm": 1.2564865350723267,
+      "learning_rate": 1.4508282807895118e-05,
+      "loss": 0.4385,
+      "step": 595
+    },
+    {
+      "epoch": 1.6224627875507442,
+      "grad_norm": 1.2202726602554321,
+      "learning_rate": 1.4284917964584706e-05,
+      "loss": 0.391,
+      "step": 600
+    },
+    {
+      "epoch": 1.6359945872801083,
+      "grad_norm": 1.2992454767227173,
+      "learning_rate": 1.4061711941702159e-05,
+      "loss": 0.4747,
+      "step": 605
+    },
+    {
+      "epoch": 1.6495263870094723,
+      "grad_norm": 1.1894512176513672,
+      "learning_rate": 1.3838714313526098e-05,
+      "loss": 0.3783,
+      "step": 610
+    },
+    {
+      "epoch": 1.6630581867388363,
+      "grad_norm": 1.0729334354400635,
+      "learning_rate": 1.3615974608050472e-05,
+      "loss": 0.4069,
+      "step": 615
+    },
+    {
+      "epoch": 1.6765899864682003,
+      "grad_norm": 1.4220861196517944,
+      "learning_rate": 1.3393542295984333e-05,
+      "loss": 0.4075,
+      "step": 620
+    },
+    {
+      "epoch": 1.6901217861975644,
+      "grad_norm": 1.1365106105804443,
+      "learning_rate": 1.317146677976436e-05,
+      "loss": 0.4028,
+      "step": 625
+    },
+    {
+      "epoch": 1.7036535859269284,
+      "grad_norm": 1.1551275253295898,
+      "learning_rate": 1.2949797382582523e-05,
+      "loss": 0.4001,
+      "step": 630
+    },
+    {
+      "epoch": 1.7171853856562924,
+      "grad_norm": 1.3404793739318848,
+      "learning_rate": 1.2728583337431355e-05,
+      "loss": 0.3909,
+      "step": 635
+    },
+    {
+      "epoch": 1.7307171853856564,
+      "grad_norm": 1.2815924882888794,
+      "learning_rate": 1.2507873776169231e-05,
+      "loss": 0.4106,
+      "step": 640
+    },
+    {
+      "epoch": 1.7442489851150202,
+      "grad_norm": 1.2801376581192017,
+      "learning_rate": 1.228771771860818e-05,
+      "loss": 0.3606,
+      "step": 645
+    },
+    {
+      "epoch": 1.7577807848443843,
+      "grad_norm": 1.2058384418487549,
+      "learning_rate": 1.2068164061626498e-05,
+      "loss": 0.3469,
+      "step": 650
+    },
+    {
+      "epoch": 1.7713125845737483,
+      "grad_norm": 1.0217390060424805,
+      "learning_rate": 1.1849261568308726e-05,
+      "loss": 0.3892,
+      "step": 655
+    },
+    {
+      "epoch": 1.7848443843031123,
+      "grad_norm": 1.1438614130020142,
+      "learning_rate": 1.1631058857115297e-05,
+      "loss": 0.3778,
+      "step": 660
+    },
+    {
+      "epoch": 1.7983761840324763,
+      "grad_norm": 1.1077343225479126,
+      "learning_rate": 1.141360439108435e-05,
+      "loss": 0.387,
+      "step": 665
+    },
+    {
+      "epoch": 1.8119079837618404,
+      "grad_norm": 1.2385257482528687,
+      "learning_rate": 1.1196946467068038e-05,
+      "loss": 0.3629,
+      "step": 670
+    },
+    {
+      "epoch": 1.8254397834912042,
+      "grad_norm": 1.0686794519424438,
+      "learning_rate": 1.0981133205005732e-05,
+      "loss": 0.3368,
+      "step": 675
+    },
+    {
+      "epoch": 1.8389715832205682,
+      "grad_norm": 1.2413495779037476,
+      "learning_rate": 1.076621253723655e-05,
+      "loss": 0.3285,
+      "step": 680
+    },
+    {
+      "epoch": 1.8525033829499322,
+      "grad_norm": 1.096193552017212,
+      "learning_rate": 1.0552232197853538e-05,
+      "loss": 0.3534,
+      "step": 685
+    },
+    {
+      "epoch": 1.8660351826792962,
+      "grad_norm": 1.287257194519043,
+      "learning_rate": 1.0339239712101875e-05,
+      "loss": 0.3511,
+      "step": 690
+    },
+    {
+      "epoch": 1.8795669824086603,
+      "grad_norm": 1.093516230583191,
+      "learning_rate": 1.0127282385823478e-05,
+      "loss": 0.3532,
+      "step": 695
+    },
+    {
+      "epoch": 1.8930987821380243,
+      "grad_norm": 1.0753506422042847,
+      "learning_rate": 9.916407294950304e-06,
+      "loss": 0.3604,
+      "step": 700
+    },
+    {
+      "epoch": 1.9066305818673883,
+      "grad_norm": 1.2421495914459229,
+      "learning_rate": 9.706661275048774e-06,
+      "loss": 0.3804,
+      "step": 705
+    },
+    {
+      "epoch": 1.9201623815967523,
+      "grad_norm": 1.1781210899353027,
+      "learning_rate": 9.498090910917508e-06,
+      "loss": 0.3698,
+      "step": 710
+    },
+    {
+      "epoch": 1.9336941813261164,
+      "grad_norm": 1.3023509979248047,
+      "learning_rate": 9.290742526240834e-06,
+      "loss": 0.4185,
+      "step": 715
+    },
+    {
+      "epoch": 1.9472259810554804,
+      "grad_norm": 1.2568118572235107,
+      "learning_rate": 9.084662173300224e-06,
+      "loss": 0.3957,
+      "step": 720
+    },
+    {
+      "epoch": 1.9607577807848444,
+      "grad_norm": 1.1244642734527588,
+      "learning_rate": 8.879895622746049e-06,
+      "loss": 0.3698,
+      "step": 725
+    },
+    {
+      "epoch": 1.9742895805142084,
+      "grad_norm": 1.3398619890213013,
+      "learning_rate": 8.676488353431868e-06,
+      "loss": 0.351,
+      "step": 730
+    },
+    {
+      "epoch": 1.9878213802435725,
+      "grad_norm": 1.0850826501846313,
+      "learning_rate": 8.474485542313535e-06,
+      "loss": 0.3253,
+      "step": 735
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.4716168642044067,
+      "learning_rate": 8.27393205441535e-06,
+      "loss": 0.3573,
+      "step": 740
+    },
+    {
+      "epoch": 2.013531799729364,
+      "grad_norm": 1.0745917558670044,
+      "learning_rate": 8.074872432865477e-06,
+      "loss": 0.258,
+      "step": 745
+    },
+    {
+      "epoch": 2.027063599458728,
+      "grad_norm": 1.0333101749420166,
+      "learning_rate": 7.877350889002908e-06,
+      "loss": 0.333,
+      "step": 750
+    },
+    {
+      "epoch": 2.040595399188092,
+      "grad_norm": 1.1765122413635254,
+      "learning_rate": 7.681411292558035e-06,
+      "loss": 0.2837,
+      "step": 755
+    },
+    {
+      "epoch": 2.054127198917456,
+      "grad_norm": 1.1787981986999512,
+      "learning_rate": 7.487097161909175e-06,
+      "loss": 0.276,
+      "step": 760
+    },
+    {
+      "epoch": 2.06765899864682,
+      "grad_norm": 1.2039293050765991,
+      "learning_rate": 7.2944516544170845e-06,
+      "loss": 0.3107,
+      "step": 765
+    },
+    {
+      "epoch": 2.081190798376184,
+      "grad_norm": 1.1785304546356201,
+      "learning_rate": 7.103517556839661e-06,
+      "loss": 0.3222,
+      "step": 770
+    },
+    {
+      "epoch": 2.094722598105548,
+      "grad_norm": 1.1416910886764526,
+      "learning_rate": 6.914337275828969e-06,
+      "loss": 0.2636,
+      "step": 775
+    },
+    {
+      "epoch": 2.108254397834912,
+      "grad_norm": 1.0554375648498535,
+      "learning_rate": 6.726952828512677e-06,
+      "loss": 0.2779,
+      "step": 780
+    },
+    {
+      "epoch": 2.121786197564276,
+      "grad_norm": 1.2561558485031128,
+      "learning_rate": 6.5414058331620036e-06,
+      "loss": 0.278,
+      "step": 785
+    },
+    {
+      "epoch": 2.1353179972936402,
+      "grad_norm": 1.0535979270935059,
+      "learning_rate": 6.3577374999482855e-06,
+      "loss": 0.2869,
+      "step": 790
+    },
+    {
+      "epoch": 2.1488497970230043,
+      "grad_norm": 1.2939218282699585,
+      "learning_rate": 6.175988621790115e-06,
+      "loss": 0.3087,
+      "step": 795
+    },
+    {
+      "epoch": 2.1623815967523683,
+      "grad_norm": 1.3614039421081543,
+      "learning_rate": 5.996199565293236e-06,
+      "loss": 0.2816,
+      "step": 800
+    },
+    {
+      "epoch": 2.175913396481732,
+      "grad_norm": 1.1801762580871582,
+      "learning_rate": 5.818410261785056e-06,
+      "loss": 0.2517,
+      "step": 805
+    },
+    {
+      "epoch": 2.189445196211096,
+      "grad_norm": 1.1439168453216553,
+      "learning_rate": 5.6426601984458655e-06,
+      "loss": 0.2826,
+      "step": 810
+    },
+    {
+      "epoch": 2.20297699594046,
+      "grad_norm": 1.1426571607589722,
+      "learning_rate": 5.468988409538695e-06,
+      "loss": 0.2899,
+      "step": 815
+    },
+    {
+      "epoch": 2.216508795669824,
+      "grad_norm": 1.216961145401001,
+      "learning_rate": 5.297433467739757e-06,
+      "loss": 0.2895,
+      "step": 820
+    },
+    {
+      "epoch": 2.230040595399188,
+      "grad_norm": 1.2847660779953003,
+      "learning_rate": 5.128033475571437e-06,
+      "loss": 0.2628,
+      "step": 825
+    },
+    {
+      "epoch": 2.243572395128552,
+      "grad_norm": 1.3062347173690796,
+      "learning_rate": 4.960826056939678e-06,
+      "loss": 0.3054,
+      "step": 830
+    },
+    {
+      "epoch": 2.257104194857916,
+      "grad_norm": 1.1694371700286865,
+      "learning_rate": 4.795848348777652e-06,
+      "loss": 0.2542,
+      "step": 835
+    },
+    {
+      "epoch": 2.27063599458728,
+      "grad_norm": 1.1245782375335693,
+      "learning_rate": 4.633136992797645e-06,
+      "loss": 0.2555,
+      "step": 840
+    },
+    {
+      "epoch": 2.284167794316644,
+      "grad_norm": 1.1038490533828735,
+      "learning_rate": 4.472728127352879e-06,
+      "loss": 0.2899,
+      "step": 845
+    },
+    {
+      "epoch": 2.297699594046008,
+      "grad_norm": 1.0687370300292969,
+      "learning_rate": 4.314657379411145e-06,
+      "loss": 0.2732,
+      "step": 850
+    },
+    {
+      "epoch": 2.311231393775372,
+      "grad_norm": 1.1048815250396729,
+      "learning_rate": 4.158959856642016e-06,
+      "loss": 0.2528,
+      "step": 855
+    },
+    {
+      "epoch": 2.324763193504736,
+      "grad_norm": 1.1284077167510986,
+      "learning_rate": 4.005670139619415e-06,
+      "loss": 0.2905,
+      "step": 860
+    },
+    {
+      "epoch": 2.3382949932341,
+      "grad_norm": 1.2731678485870361,
+      "learning_rate": 3.854822274141223e-06,
+      "loss": 0.2527,
+      "step": 865
+    },
+    {
+      "epoch": 2.351826792963464,
+      "grad_norm": 1.1605026721954346,
+      "learning_rate": 3.706449763667681e-06,
+      "loss": 0.2548,
+      "step": 870
+    },
+    {
+      "epoch": 2.365358592692828,
+      "grad_norm": 1.124977469444275,
+      "learning_rate": 3.5605855618802153e-06,
+      "loss": 0.2766,
+      "step": 875
+    },
+    {
+      "epoch": 2.3788903924221922,
+      "grad_norm": 1.2432584762573242,
+      "learning_rate": 3.417262065362429e-06,
+      "loss": 0.2637,
+      "step": 880
+    },
+    {
+      "epoch": 2.3924221921515563,
+      "grad_norm": 1.1530462503433228,
+      "learning_rate": 3.2765111064047663e-06,
+      "loss": 0.2931,
+      "step": 885
+    },
+    {
+      "epoch": 2.4059539918809203,
+      "grad_norm": 1.2465864419937134,
+      "learning_rate": 3.1383639459345237e-06,
+      "loss": 0.2418,
+      "step": 890
+    },
+    {
+      "epoch": 2.4194857916102843,
+      "grad_norm": 1.1902434825897217,
+      "learning_rate": 3.0028512665727845e-06,
+      "loss": 0.2532,
+      "step": 895
+    },
+    {
+      "epoch": 2.4330175913396483,
+      "grad_norm": 1.039732813835144,
+      "learning_rate": 2.8700031658197682e-06,
+      "loss": 0.2346,
+      "step": 900
+    },
+    {
+      "epoch": 2.4465493910690124,
+      "grad_norm": 1.2276580333709717,
+      "learning_rate": 2.7398491493701503e-06,
+      "loss": 0.2938,
+      "step": 905
+    },
+    {
+      "epoch": 2.460081190798376,
+      "grad_norm": 1.2137160301208496,
+      "learning_rate": 2.6124181245598145e-06,
+      "loss": 0.2682,
+      "step": 910
+    },
+    {
+      "epoch": 2.47361299052774,
+      "grad_norm": 1.142590880393982,
+      "learning_rate": 2.4877383939454986e-06,
+      "loss": 0.256,
+      "step": 915
+    },
+    {
+      "epoch": 2.487144790257104,
+      "grad_norm": 1.1829429864883423,
+      "learning_rate": 2.3658376490187905e-06,
+      "loss": 0.2687,
+      "step": 920
+    },
+    {
+      "epoch": 2.500676589986468,
+      "grad_norm": 1.090276837348938,
+      "learning_rate": 2.2467429640557903e-06,
+      "loss": 0.3007,
+      "step": 925
+    },
+    {
+      "epoch": 2.514208389715832,
+      "grad_norm": 1.1636515855789185,
+      "learning_rate": 2.1304807901039146e-06,
+      "loss": 0.2461,
+      "step": 930
+    },
+    {
+      "epoch": 2.527740189445196,
+      "grad_norm": 1.0615500211715698,
+      "learning_rate": 2.017076949107077e-06,
+      "loss": 0.2607,
+      "step": 935
+    },
+    {
+      "epoch": 2.54127198917456,
+      "grad_norm": 1.3706880807876587,
+      "learning_rate": 1.9065566281706126e-06,
+      "loss": 0.2633,
+      "step": 940
+    },
+    {
+      "epoch": 2.554803788903924,
+      "grad_norm": 1.3997299671173096,
+      "learning_rate": 1.7989443739671968e-06,
+      "loss": 0.2581,
+      "step": 945
+    },
+    {
+      "epoch": 2.568335588633288,
+      "grad_norm": 1.1273524761199951,
+      "learning_rate": 1.6942640872850085e-06,
+      "loss": 0.2504,
+      "step": 950
+    },
+    {
+      "epoch": 2.581867388362652,
+      "grad_norm": 1.1891227960586548,
+      "learning_rate": 1.5925390177193266e-06,
+      "loss": 0.2586,
+      "step": 955
+    },
+    {
+      "epoch": 2.595399188092016,
+      "grad_norm": 1.1752684116363525,
+      "learning_rate": 1.4937917585087978e-06,
+      "loss": 0.2741,
+      "step": 960
+    },
+    {
+      "epoch": 2.60893098782138,
+      "grad_norm": 1.2403773069381714,
+      "learning_rate": 1.3980442415174305e-06,
+      "loss": 0.2837,
+      "step": 965
+    },
+    {
+      "epoch": 2.6224627875507442,
+      "grad_norm": 1.3768302202224731,
+      "learning_rate": 1.3053177323635224e-06,
+      "loss": 0.2637,
+      "step": 970
+    },
+    {
+      "epoch": 2.6359945872801083,
+      "grad_norm": 1.1108583211898804,
+      "learning_rate": 1.215632825696541e-06,
+      "loss": 0.2642,
+      "step": 975
+    },
+    {
+      "epoch": 2.6495263870094723,
+      "grad_norm": 1.1133606433868408,
+      "learning_rate": 1.1290094406230273e-06,
+      "loss": 0.2897,
+      "step": 980
+    },
+    {
+      "epoch": 2.6630581867388363,
+      "grad_norm": 1.0985251665115356,
+      "learning_rate": 1.045466816282551e-06,
+      "loss": 0.2669,
+      "step": 985
+    },
+    {
+      "epoch": 2.6765899864682003,
+      "grad_norm": 1.1606647968292236,
+      "learning_rate": 9.650235075746688e-07,
+      "loss": 0.2792,
+      "step": 990
+    },
+    {
+      "epoch": 2.6901217861975644,
+      "grad_norm": 1.3157551288604736,
+      "learning_rate": 8.876973810378663e-07,
+      "loss": 0.2526,
+      "step": 995
+    },
+    {
+      "epoch": 2.7036535859269284,
+      "grad_norm": 1.1697179079055786,
+      "learning_rate": 8.135056108813898e-07,
+      "loss": 0.2549,
+      "step": 1000
+    },
+    {
+      "epoch": 2.7171853856562924,
+      "grad_norm": 1.2197825908660889,
+      "learning_rate": 7.424646751708247e-07,
+      "loss": 0.2762,
+      "step": 1005
+    },
+    {
+      "epoch": 2.7307171853856564,
+      "grad_norm": 1.2570303678512573,
+      "learning_rate": 6.745903521683128e-07,
+      "loss": 0.2616,
+      "step": 1010
+    },
+    {
+      "epoch": 2.7442489851150205,
+      "grad_norm": 1.1606673002243042,
+      "learning_rate": 6.098977168281738e-07,
+      "loss": 0.2662,
+      "step": 1015
+    },
+    {
+      "epoch": 2.7577807848443845,
+      "grad_norm": 1.5005513429641724,
+      "learning_rate": 5.48401137448754e-07,
+      "loss": 0.2458,
+      "step": 1020
+    },
+    {
+      "epoch": 2.7713125845737485,
+      "grad_norm": 1.0978503227233887,
+      "learning_rate": 4.901142724812019e-07,
+      "loss": 0.2978,
+      "step": 1025
+    },
+    {
+      "epoch": 2.7848443843031125,
+      "grad_norm": 0.9531242251396179,
+      "learning_rate": 4.350500674959212e-07,
+      "loss": 0.2601,
+      "step": 1030
+    },
+    {
+      "epoch": 2.7983761840324766,
+      "grad_norm": 1.1144373416900635,
+      "learning_rate": 3.832207523073433e-07,
+      "loss": 0.239,
+      "step": 1035
+    },
+    {
+      "epoch": 2.8119079837618406,
+      "grad_norm": 1.1502642631530762,
+      "learning_rate": 3.346378382576726e-07,
+      "loss": 0.2471,
+      "step": 1040
+    },
+    {
+      "epoch": 2.825439783491204,
+      "grad_norm": 1.0708447694778442,
+      "learning_rate": 2.8931211566020854e-07,
+      "loss": 0.2587,
+      "step": 1045
+    },
+    {
+      "epoch": 2.838971583220568,
+      "grad_norm": 1.1705080270767212,
+      "learning_rate": 2.4725365140280123e-07,
+      "loss": 0.2606,
+      "step": 1050
+    },
+    {
+      "epoch": 2.852503382949932,
+      "grad_norm": 1.2843471765518188,
+      "learning_rate": 2.0847178671199785e-07,
+      "loss": 0.2853,
+      "step": 1055
+    },
+    {
+      "epoch": 2.8660351826792962,
+      "grad_norm": 1.0960692167282104,
+      "learning_rate": 1.729751350783293e-07,
+      "loss": 0.2352,
+      "step": 1060
+    },
+    {
+      "epoch": 2.8795669824086603,
+      "grad_norm": 1.2112921476364136,
+      "learning_rate": 1.407715803432602e-07,
+      "loss": 0.2351,
+      "step": 1065
+    },
+    {
+      "epoch": 2.8930987821380243,
+      "grad_norm": 1.1370913982391357,
+      "learning_rate": 1.1186827494817308e-07,
+      "loss": 0.2542,
+      "step": 1070
+    },
+    {
+      "epoch": 2.9066305818673883,
+      "grad_norm": 1.1701987981796265,
+      "learning_rate": 8.627163834580565e-08,
+      "loss": 0.2768,
+      "step": 1075
+    },
+    {
+      "epoch": 2.9201623815967523,
+      "grad_norm": 1.027894139289856,
+      "learning_rate": 6.398735557448299e-08,
+      "loss": 0.2613,
+      "step": 1080
+    },
+    {
+      "epoch": 2.9336941813261164,
+      "grad_norm": 1.1382505893707275,
+      "learning_rate": 4.502037599546427e-08,
+      "loss": 0.2482,
+      "step": 1085
+    },
+    {
+      "epoch": 2.9472259810554804,
+      "grad_norm": 1.1618467569351196,
+      "learning_rate": 2.9374912193690374e-08,
+      "loss": 0.2628,
+      "step": 1090
+    },
+    {
+      "epoch": 2.9607577807848444,
+      "grad_norm": 1.1010104417800903,
+      "learning_rate": 1.705443904215731e-08,
+      "loss": 0.2317,
+      "step": 1095
+    },
+    {
+      "epoch": 2.9742895805142084,
+      "grad_norm": 1.117809534072876,
+      "learning_rate": 8.061692930146913e-09,
+      "loss": 0.2607,
+      "step": 1100
+    },
+    {
+      "epoch": 2.9878213802435725,
+      "grad_norm": 1.1122850179672241,
+      "learning_rate": 2.3986711554713215e-09,
+      "loss": 0.2537,
+      "step": 1105
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 1.3885308504104614,
+      "learning_rate": 6.663148087282522e-11,
+      "loss": 0.241,
+      "step": 1110
+    },
+    {
+      "epoch": 3.0,
+      "step": 1110,
+      "total_flos": 1.619647455794561e+18,
+      "train_loss": 0.537263454939868,
+      "train_runtime": 774.41,
+      "train_samples_per_second": 45.778,
+      "train_steps_per_second": 1.433
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1110,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.619647455794561e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

12_128_e3_3e-5/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ad513382674ce81cbdd3b56f5db3489cc76711437808e7f7bf9444d0bcad00a
+size 8145