Upload folder using huggingface_hub

README.md

@@ -0,0 +1,202 @@
+---
+base_model: unsloth/Phi-3.5-mini-instruct-bnb-4bit
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/Phi-3.5-mini-instruct-bnb-4bit",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33b37a802426a88fbfd4cf8085fb78f99e5b23005e7fc50ef4761443261891da
+size 239135488

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc84e43d31602542ed7beda098fc71347a2d673e71e93311543cc266520ce0d3
+size 121933285

projection_layer.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eeb7cab86f933cddcd712bd2056c6e51e6d57bf331251ffa04d50d29d366135c
+size 62937069

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70c280155fd4431174c846286a1a935695840e03f6c7c0c0fcde8619b978fac5
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9b0adcccbed00748a79952459310c319a4aac904c4e3f6184127dc73271b74e
+size 1465

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|placeholder6|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": true,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,132 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 131072,
+  "pad_token": "<|placeholder6|>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,793 @@
+{
+  "best_global_step": 50000,
+  "best_metric": 1.1446055173873901,
+  "best_model_checkpoint": "output/phi3/phi3.5_mini_rkd/checkpoint-50000",
+  "epoch": 5.903267703751808,
+  "eval_steps": 1000,
+  "global_step": 50000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.11807420964075922,
+      "grad_norm": 1.2256360054016113,
+      "learning_rate": 0.0001980437499384982,
+      "loss": 41.6736,
+      "step": 1000
+    },
+    {
+      "epoch": 0.11807420964075922,
+      "eval_loss": 1.5498648881912231,
+      "eval_runtime": 25.7697,
+      "eval_samples_per_second": 116.416,
+      "eval_steps_per_second": 7.295,
+      "step": 1000
+    },
+    {
+      "epoch": 0.23614841928151845,
+      "grad_norm": 3.2936487197875977,
+      "learning_rate": 0.00019607569152652453,
+      "loss": 41.7768,
+      "step": 2000
+    },
+    {
+      "epoch": 0.23614841928151845,
+      "eval_loss": 1.4593058824539185,
+      "eval_runtime": 24.0365,
+      "eval_samples_per_second": 124.81,
+      "eval_steps_per_second": 7.821,
+      "step": 2000
+    },
+    {
+      "epoch": 0.3542226289222776,
+      "grad_norm": 1.248852252960205,
+      "learning_rate": 0.00019410763311455083,
+      "loss": 41.9198,
+      "step": 3000
+    },
+    {
+      "epoch": 0.3542226289222776,
+      "eval_loss": 1.4187239408493042,
+      "eval_runtime": 24.1485,
+      "eval_samples_per_second": 124.231,
+      "eval_steps_per_second": 7.785,
+      "step": 3000
+    },
+    {
+      "epoch": 0.4722968385630369,
+      "grad_norm": 3.8973965644836426,
+      "learning_rate": 0.00019213957470257716,
+      "loss": 41.3329,
+      "step": 4000
+    },
+    {
+      "epoch": 0.4722968385630369,
+      "eval_loss": 1.3792623281478882,
+      "eval_runtime": 24.9602,
+      "eval_samples_per_second": 120.191,
+      "eval_steps_per_second": 7.532,
+      "step": 4000
+    },
+    {
+      "epoch": 0.590371048203796,
+      "grad_norm": 5.706345081329346,
+      "learning_rate": 0.00019017151629060352,
+      "loss": 41.2385,
+      "step": 5000
+    },
+    {
+      "epoch": 0.590371048203796,
+      "eval_loss": 1.3622702360153198,
+      "eval_runtime": 24.0781,
+      "eval_samples_per_second": 124.595,
+      "eval_steps_per_second": 7.808,
+      "step": 5000
+    },
+    {
+      "epoch": 0.7084452578445553,
+      "grad_norm": 2.7865254878997803,
+      "learning_rate": 0.00018820345787862986,
+      "loss": 41.2127,
+      "step": 6000
+    },
+    {
+      "epoch": 0.7084452578445553,
+      "eval_loss": 1.3430252075195312,
+      "eval_runtime": 23.7305,
+      "eval_samples_per_second": 126.42,
+      "eval_steps_per_second": 7.922,
+      "step": 6000
+    },
+    {
+      "epoch": 0.8265194674853146,
+      "grad_norm": 5.064664840698242,
+      "learning_rate": 0.0001862353994666562,
+      "loss": 41.6429,
+      "step": 7000
+    },
+    {
+      "epoch": 0.8265194674853146,
+      "eval_loss": 1.3215986490249634,
+      "eval_runtime": 24.6492,
+      "eval_samples_per_second": 121.708,
+      "eval_steps_per_second": 7.627,
+      "step": 7000
+    },
+    {
+      "epoch": 0.9445936771260738,
+      "grad_norm": 3.32861590385437,
+      "learning_rate": 0.00018426734105468252,
+      "loss": 41.0908,
+      "step": 8000
+    },
+    {
+      "epoch": 0.9445936771260738,
+      "eval_loss": 1.3072400093078613,
+      "eval_runtime": 23.9184,
+      "eval_samples_per_second": 125.426,
+      "eval_steps_per_second": 7.86,
+      "step": 8000
+    },
+    {
+      "epoch": 1.0625793311096023,
+      "grad_norm": 0.9595094919204712,
+      "learning_rate": 0.00018229928264270882,
+      "loss": 40.9293,
+      "step": 9000
+    },
+    {
+      "epoch": 1.0625793311096023,
+      "eval_loss": 1.2909538745880127,
+      "eval_runtime": 25.2926,
+      "eval_samples_per_second": 118.612,
+      "eval_steps_per_second": 7.433,
+      "step": 9000
+    },
+    {
+      "epoch": 1.1806535407503616,
+      "grad_norm": 2.542973041534424,
+      "learning_rate": 0.00018033122423073518,
+      "loss": 41.0413,
+      "step": 10000
+    },
+    {
+      "epoch": 1.1806535407503616,
+      "eval_loss": 1.2891528606414795,
+      "eval_runtime": 25.8904,
+      "eval_samples_per_second": 115.873,
+      "eval_steps_per_second": 7.261,
+      "step": 10000
+    },
+    {
+      "epoch": 1.2987277503911208,
+      "grad_norm": 1.4823276996612549,
+      "learning_rate": 0.0001783631658187615,
+      "loss": 41.0681,
+      "step": 11000
+    },
+    {
+      "epoch": 1.2987277503911208,
+      "eval_loss": 1.2768892049789429,
+      "eval_runtime": 24.7234,
+      "eval_samples_per_second": 121.343,
+      "eval_steps_per_second": 7.604,
+      "step": 11000
+    },
+    {
+      "epoch": 1.41680196003188,
+      "grad_norm": 1.6149150133132935,
+      "learning_rate": 0.00017639510740678784,
+      "loss": 40.9333,
+      "step": 12000
+    },
+    {
+      "epoch": 1.41680196003188,
+      "eval_loss": 1.265001654624939,
+      "eval_runtime": 26.8828,
+      "eval_samples_per_second": 111.596,
+      "eval_steps_per_second": 6.993,
+      "step": 12000
+    },
+    {
+      "epoch": 1.5348761696726392,
+      "grad_norm": 1.3511199951171875,
+      "learning_rate": 0.00017442704899481418,
+      "loss": 41.1986,
+      "step": 13000
+    },
+    {
+      "epoch": 1.5348761696726392,
+      "eval_loss": 1.2615211009979248,
+      "eval_runtime": 24.2215,
+      "eval_samples_per_second": 123.857,
+      "eval_steps_per_second": 7.762,
+      "step": 13000
+    },
+    {
+      "epoch": 1.6529503793133986,
+      "grad_norm": 2.274367570877075,
+      "learning_rate": 0.0001724589905828405,
+      "loss": 40.5442,
+      "step": 14000
+    },
+    {
+      "epoch": 1.6529503793133986,
+      "eval_loss": 1.245273232460022,
+      "eval_runtime": 24.3365,
+      "eval_samples_per_second": 123.272,
+      "eval_steps_per_second": 7.725,
+      "step": 14000
+    },
+    {
+      "epoch": 1.7710245889541576,
+      "grad_norm": 2.070169687271118,
+      "learning_rate": 0.00017049093217086684,
+      "loss": 41.5188,
+      "step": 15000
+    },
+    {
+      "epoch": 1.7710245889541576,
+      "eval_loss": 1.2442792654037476,
+      "eval_runtime": 24.3005,
+      "eval_samples_per_second": 123.454,
+      "eval_steps_per_second": 7.736,
+      "step": 15000
+    },
+    {
+      "epoch": 1.889098798594917,
+      "grad_norm": 11.877659797668457,
+      "learning_rate": 0.00016852287375889317,
+      "loss": 41.0519,
+      "step": 16000
+    },
+    {
+      "epoch": 1.889098798594917,
+      "eval_loss": 1.2491201162338257,
+      "eval_runtime": 24.021,
+      "eval_samples_per_second": 124.891,
+      "eval_steps_per_second": 7.826,
+      "step": 16000
+    },
+    {
+      "epoch": 2.0070844525784457,
+      "grad_norm": 7.578474044799805,
+      "learning_rate": 0.0001665548153469195,
+      "loss": 40.4365,
+      "step": 17000
+    },
+    {
+      "epoch": 2.0070844525784457,
+      "eval_loss": 1.2308341264724731,
+      "eval_runtime": 24.4199,
+      "eval_samples_per_second": 122.851,
+      "eval_steps_per_second": 7.699,
+      "step": 17000
+    },
+    {
+      "epoch": 2.1251586622192047,
+      "grad_norm": 1.8282848596572876,
+      "learning_rate": 0.00016458675693494583,
+      "loss": 40.5926,
+      "step": 18000
+    },
+    {
+      "epoch": 2.1251586622192047,
+      "eval_loss": 1.2286590337753296,
+      "eval_runtime": 24.0718,
+      "eval_samples_per_second": 124.627,
+      "eval_steps_per_second": 7.81,
+      "step": 18000
+    },
+    {
+      "epoch": 2.243232871859964,
+      "grad_norm": 3.1526448726654053,
+      "learning_rate": 0.0001626186985229722,
+      "loss": 40.5593,
+      "step": 19000
+    },
+    {
+      "epoch": 2.243232871859964,
+      "eval_loss": 1.2166979312896729,
+      "eval_runtime": 24.1582,
+      "eval_samples_per_second": 124.181,
+      "eval_steps_per_second": 7.782,
+      "step": 19000
+    },
+    {
+      "epoch": 2.361307081500723,
+      "grad_norm": 1.2013620138168335,
+      "learning_rate": 0.0001606506401109985,
+      "loss": 40.6833,
+      "step": 20000
+    },
+    {
+      "epoch": 2.361307081500723,
+      "eval_loss": 1.2144083976745605,
+      "eval_runtime": 24.2637,
+      "eval_samples_per_second": 123.641,
+      "eval_steps_per_second": 7.748,
+      "step": 20000
+    },
+    {
+      "epoch": 2.4793812911414825,
+      "grad_norm": 9.406246185302734,
+      "learning_rate": 0.00015868258169902483,
+      "loss": 40.9452,
+      "step": 21000
+    },
+    {
+      "epoch": 2.4793812911414825,
+      "eval_loss": 1.209915041923523,
+      "eval_runtime": 24.0813,
+      "eval_samples_per_second": 124.578,
+      "eval_steps_per_second": 7.807,
+      "step": 21000
+    },
+    {
+      "epoch": 2.5974555007822415,
+      "grad_norm": 3.906858444213867,
+      "learning_rate": 0.00015671452328705116,
+      "loss": 40.6357,
+      "step": 22000
+    },
+    {
+      "epoch": 2.5974555007822415,
+      "eval_loss": 1.2079243659973145,
+      "eval_runtime": 24.1172,
+      "eval_samples_per_second": 124.392,
+      "eval_steps_per_second": 7.795,
+      "step": 22000
+    },
+    {
+      "epoch": 2.715529710423001,
+      "grad_norm": 2.3013319969177246,
+      "learning_rate": 0.0001547464648750775,
+      "loss": 40.9317,
+      "step": 23000
+    },
+    {
+      "epoch": 2.715529710423001,
+      "eval_loss": 1.2017567157745361,
+      "eval_runtime": 23.9287,
+      "eval_samples_per_second": 125.373,
+      "eval_steps_per_second": 7.857,
+      "step": 23000
+    },
+    {
+      "epoch": 2.83360392006376,
+      "grad_norm": 1.8648923635482788,
+      "learning_rate": 0.00015277840646310385,
+      "loss": 40.7994,
+      "step": 24000
+    },
+    {
+      "epoch": 2.83360392006376,
+      "eval_loss": 1.1976925134658813,
+      "eval_runtime": 23.79,
+      "eval_samples_per_second": 126.104,
+      "eval_steps_per_second": 7.902,
+      "step": 24000
+    },
+    {
+      "epoch": 2.9516781297045194,
+      "grad_norm": 4.949862480163574,
+      "learning_rate": 0.00015081034805113018,
+      "loss": 40.7059,
+      "step": 25000
+    },
+    {
+      "epoch": 2.9516781297045194,
+      "eval_loss": 1.191980242729187,
+      "eval_runtime": 23.9475,
+      "eval_samples_per_second": 125.274,
+      "eval_steps_per_second": 7.851,
+      "step": 25000
+    },
+    {
+      "epoch": 3.069663783688048,
+      "grad_norm": 2.596585988998413,
+      "learning_rate": 0.00014884228963915649,
+      "loss": 40.6073,
+      "step": 26000
+    },
+    {
+      "epoch": 3.069663783688048,
+      "eval_loss": 1.1963772773742676,
+      "eval_runtime": 23.9925,
+      "eval_samples_per_second": 125.039,
+      "eval_steps_per_second": 7.836,
+      "step": 26000
+    },
+    {
+      "epoch": 3.187737993328807,
+      "grad_norm": 5.206244468688965,
+      "learning_rate": 0.00014687423122718282,
+      "loss": 40.6736,
+      "step": 27000
+    },
+    {
+      "epoch": 3.187737993328807,
+      "eval_loss": 1.1900700330734253,
+      "eval_runtime": 24.0857,
+      "eval_samples_per_second": 124.555,
+      "eval_steps_per_second": 7.805,
+      "step": 27000
+    },
+    {
+      "epoch": 3.3058122029695665,
+      "grad_norm": 2.3656837940216064,
+      "learning_rate": 0.00014490617281520915,
+      "loss": 40.5928,
+      "step": 28000
+    },
+    {
+      "epoch": 3.3058122029695665,
+      "eval_loss": 1.1859327554702759,
+      "eval_runtime": 24.2154,
+      "eval_samples_per_second": 123.888,
+      "eval_steps_per_second": 7.764,
+      "step": 28000
+    },
+    {
+      "epoch": 3.4238864126103254,
+      "grad_norm": 4.46853494644165,
+      "learning_rate": 0.0001429381144032355,
+      "loss": 40.7044,
+      "step": 29000
+    },
+    {
+      "epoch": 3.4238864126103254,
+      "eval_loss": 1.1856085062026978,
+      "eval_runtime": 24.1112,
+      "eval_samples_per_second": 124.423,
+      "eval_steps_per_second": 7.797,
+      "step": 29000
+    },
+    {
+      "epoch": 3.541960622251085,
+      "grad_norm": 4.09805154800415,
+      "learning_rate": 0.00014097005599126184,
+      "loss": 40.6858,
+      "step": 30000
+    },
+    {
+      "epoch": 3.541960622251085,
+      "eval_loss": 1.184323787689209,
+      "eval_runtime": 24.0204,
+      "eval_samples_per_second": 124.894,
+      "eval_steps_per_second": 7.827,
+      "step": 30000
+    },
+    {
+      "epoch": 3.660034831891844,
+      "grad_norm": 6.184741497039795,
+      "learning_rate": 0.00013900199757928814,
+      "loss": 40.4295,
+      "step": 31000
+    },
+    {
+      "epoch": 3.660034831891844,
+      "eval_loss": 1.1791220903396606,
+      "eval_runtime": 23.9084,
+      "eval_samples_per_second": 125.479,
+      "eval_steps_per_second": 7.863,
+      "step": 31000
+    },
+    {
+      "epoch": 3.7781090415326033,
+      "grad_norm": 5.132469654083252,
+      "learning_rate": 0.00013703393916731447,
+      "loss": 40.7522,
+      "step": 32000
+    },
+    {
+      "epoch": 3.7781090415326033,
+      "eval_loss": 1.1751279830932617,
+      "eval_runtime": 24.1292,
+      "eval_samples_per_second": 124.331,
+      "eval_steps_per_second": 7.791,
+      "step": 32000
+    },
+    {
+      "epoch": 3.8961832511733627,
+      "grad_norm": 1.4844293594360352,
+      "learning_rate": 0.00013506588075534083,
+      "loss": 40.351,
+      "step": 33000
+    },
+    {
+      "epoch": 3.8961832511733627,
+      "eval_loss": 1.1678744554519653,
+      "eval_runtime": 24.0038,
+      "eval_samples_per_second": 124.98,
+      "eval_steps_per_second": 7.832,
+      "step": 33000
+    },
+    {
+      "epoch": 4.014168905156891,
+      "grad_norm": 2.605137586593628,
+      "learning_rate": 0.00013309782234336717,
+      "loss": 40.1967,
+      "step": 34000
+    },
+    {
+      "epoch": 4.014168905156891,
+      "eval_loss": 1.1783897876739502,
+      "eval_runtime": 24.1036,
+      "eval_samples_per_second": 124.463,
+      "eval_steps_per_second": 7.8,
+      "step": 34000
+    },
+    {
+      "epoch": 4.13224311479765,
+      "grad_norm": 1.4578678607940674,
+      "learning_rate": 0.0001311297639313935,
+      "loss": 40.3199,
+      "step": 35000
+    },
+    {
+      "epoch": 4.13224311479765,
+      "eval_loss": 1.1700868606567383,
+      "eval_runtime": 24.2003,
+      "eval_samples_per_second": 123.965,
+      "eval_steps_per_second": 7.768,
+      "step": 35000
+    },
+    {
+      "epoch": 4.250317324438409,
+      "grad_norm": 2.9243862628936768,
+      "learning_rate": 0.00012916170551941983,
+      "loss": 40.1337,
+      "step": 36000
+    },
+    {
+      "epoch": 4.250317324438409,
+      "eval_loss": 1.1697266101837158,
+      "eval_runtime": 24.1065,
+      "eval_samples_per_second": 124.448,
+      "eval_steps_per_second": 7.799,
+      "step": 36000
+    },
+    {
+      "epoch": 4.368391534079168,
+      "grad_norm": 1.479750156402588,
+      "learning_rate": 0.00012719364710744613,
+      "loss": 40.4432,
+      "step": 37000
+    },
+    {
+      "epoch": 4.368391534079168,
+      "eval_loss": 1.1706390380859375,
+      "eval_runtime": 24.0871,
+      "eval_samples_per_second": 124.548,
+      "eval_steps_per_second": 7.805,
+      "step": 37000
+    },
+    {
+      "epoch": 4.486465743719928,
+      "grad_norm": 1.2355018854141235,
+      "learning_rate": 0.0001252255886954725,
+      "loss": 40.6522,
+      "step": 38000
+    },
+    {
+      "epoch": 4.486465743719928,
+      "eval_loss": 1.1664882898330688,
+      "eval_runtime": 24.0801,
+      "eval_samples_per_second": 124.584,
+      "eval_steps_per_second": 7.807,
+      "step": 38000
+    },
+    {
+      "epoch": 4.604539953360687,
+      "grad_norm": 3.1956660747528076,
+      "learning_rate": 0.00012325753028349882,
+      "loss": 40.3552,
+      "step": 39000
+    },
+    {
+      "epoch": 4.604539953360687,
+      "eval_loss": 1.1639021635055542,
+      "eval_runtime": 23.9182,
+      "eval_samples_per_second": 125.428,
+      "eval_steps_per_second": 7.86,
+      "step": 39000
+    },
+    {
+      "epoch": 4.722614163001446,
+      "grad_norm": 1.105310082435608,
+      "learning_rate": 0.00012128947187152515,
+      "loss": 40.1775,
+      "step": 40000
+    },
+    {
+      "epoch": 4.722614163001446,
+      "eval_loss": 1.1588863134384155,
+      "eval_runtime": 24.0827,
+      "eval_samples_per_second": 124.571,
+      "eval_steps_per_second": 7.806,
+      "step": 40000
+    },
+    {
+      "epoch": 4.840688372642205,
+      "grad_norm": 2.2787368297576904,
+      "learning_rate": 0.00011932141345955147,
+      "loss": 40.6587,
+      "step": 41000
+    },
+    {
+      "epoch": 4.840688372642205,
+      "eval_loss": 1.1549768447875977,
+      "eval_runtime": 25.4826,
+      "eval_samples_per_second": 117.728,
+      "eval_steps_per_second": 7.378,
+      "step": 41000
+    },
+    {
+      "epoch": 4.958762582282965,
+      "grad_norm": 6.428521156311035,
+      "learning_rate": 0.00011735335504757783,
+      "loss": 40.6453,
+      "step": 42000
+    },
+    {
+      "epoch": 4.958762582282965,
+      "eval_loss": 1.151072382926941,
+      "eval_runtime": 24.1281,
+      "eval_samples_per_second": 124.336,
+      "eval_steps_per_second": 7.792,
+      "step": 42000
+    },
+    {
+      "epoch": 5.076748236266494,
+      "grad_norm": 4.7501068115234375,
+      "learning_rate": 0.00011538529663560416,
+      "loss": 40.1952,
+      "step": 43000
+    },
+    {
+      "epoch": 5.076748236266494,
+      "eval_loss": 1.1602451801300049,
+      "eval_runtime": 24.2098,
+      "eval_samples_per_second": 123.917,
+      "eval_steps_per_second": 7.765,
+      "step": 43000
+    },
+    {
+      "epoch": 5.194822445907253,
+      "grad_norm": 12.156052589416504,
+      "learning_rate": 0.00011341723822363048,
+      "loss": 40.6364,
+      "step": 44000
+    },
+    {
+      "epoch": 5.194822445907253,
+      "eval_loss": 1.160738229751587,
+      "eval_runtime": 24.3182,
+      "eval_samples_per_second": 123.365,
+      "eval_steps_per_second": 7.731,
+      "step": 44000
+    },
+    {
+      "epoch": 5.312896655548012,
+      "grad_norm": 2.6712443828582764,
+      "learning_rate": 0.00011144917981165681,
+      "loss": 40.0545,
+      "step": 45000
+    },
+    {
+      "epoch": 5.312896655548012,
+      "eval_loss": 1.1579570770263672,
+      "eval_runtime": 24.2733,
+      "eval_samples_per_second": 123.592,
+      "eval_steps_per_second": 7.745,
+      "step": 45000
+    },
+    {
+      "epoch": 5.430970865188771,
+      "grad_norm": 0.9378123879432678,
+      "learning_rate": 0.00010948112139968314,
+      "loss": 40.0448,
+      "step": 46000
+    },
+    {
+      "epoch": 5.430970865188771,
+      "eval_loss": 1.1599079370498657,
+      "eval_runtime": 24.0523,
+      "eval_samples_per_second": 124.728,
+      "eval_steps_per_second": 7.816,
+      "step": 46000
+    },
+    {
+      "epoch": 5.549045074829531,
+      "grad_norm": 4.0598249435424805,
+      "learning_rate": 0.00010751306298770949,
+      "loss": 40.3088,
+      "step": 47000
+    },
+    {
+      "epoch": 5.549045074829531,
+      "eval_loss": 1.1567158699035645,
+      "eval_runtime": 25.7906,
+      "eval_samples_per_second": 116.322,
+      "eval_steps_per_second": 7.289,
+      "step": 47000
+    },
+    {
+      "epoch": 5.66711928447029,
+      "grad_norm": 1.2934672832489014,
+      "learning_rate": 0.00010554500457573582,
+      "loss": 40.3503,
+      "step": 48000
+    },
+    {
+      "epoch": 5.66711928447029,
+      "eval_loss": 1.1556658744812012,
+      "eval_runtime": 24.2988,
+      "eval_samples_per_second": 123.463,
+      "eval_steps_per_second": 7.737,
+      "step": 48000
+    },
+    {
+      "epoch": 5.7851934941110486,
+      "grad_norm": 1.3672287464141846,
+      "learning_rate": 0.00010357694616376215,
+      "loss": 40.4187,
+      "step": 49000
+    },
+    {
+      "epoch": 5.7851934941110486,
+      "eval_loss": 1.1513793468475342,
+      "eval_runtime": 24.2727,
+      "eval_samples_per_second": 123.596,
+      "eval_steps_per_second": 7.745,
+      "step": 49000
+    },
+    {
+      "epoch": 5.903267703751808,
+      "grad_norm": 1.8149452209472656,
+      "learning_rate": 0.00010160888775178847,
+      "loss": 40.15,
+      "step": 50000
+    },
+    {
+      "epoch": 5.903267703751808,
+      "eval_loss": 1.1446055173873901,
+      "eval_runtime": 24.2774,
+      "eval_samples_per_second": 123.572,
+      "eval_steps_per_second": 7.744,
+      "step": 50000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 101628,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 12,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 10,
+        "early_stopping_threshold": 0.0001
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.4591371305066906e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90dd21b7eca93f81de48353c4e5f1767e79ea59501db3cdedabb214026aa5a4d
+size 6097