Add files using upload-large-folder tool

README.md

@@ -1,3 +1,202 @@
----
-license: mit
----
+---
+base_model: naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a6551b8c58ce9d8961031fb6846b0764b6df04c21e7a050afc85c235ceb7be
+size 39366152

added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1400/README.md

@@ -0,0 +1,202 @@
+---
+base_model: naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

checkpoint-1400/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-1400/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

checkpoint-1400/chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:234ba2ba854e17783128a343692bbffa9b7636ddfe7803957b3de9cafbbfa181
+size 14244

checkpoint-1400/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baba31a5e5063037a5c811de9cb04bc62c6c5f0f5fe6720b7d681afe6500d4c1
+size 988

checkpoint-1400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:365dc177b1fbdd30dc6e482006022921a3dc54e0f34f35f416f0bb1c7f192633
+size 1064

checkpoint-1400/special_tokens_map.json

@@ -0,0 +1,86 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endofturn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1400/tokenizer_config.json

@@ -0,0 +1,501 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|_unuse_missing_100256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|_unuse_missing_100261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|_unuse_missing_100262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|_unuse_missing_100263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|_unuse_missing_100264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|_unuse_missing_100265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|_unuse_missing_100266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|_unuse_missing_100267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|_unuse_missing_100268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|_unuse_missing_100269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|_unuse_missing_100270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|_unuse_missing_100271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|stop|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|endofturn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110491": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110492": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110493": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110494": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110495": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110496": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110497": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110498": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110499": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110500": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110501": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110502": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110503": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110504": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110505": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110506": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110507": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110508": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110509": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110510": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110511": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110512": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110513": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110514": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110515": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110516": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110517": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110518": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110519": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110520": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110521": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110522": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110523": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endofturn|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1400/trainer_state.json

@@ -0,0 +1,1364 @@
+{
+  "best_global_step": 1000,
+  "best_metric": 1.8698750734329224,
+  "best_model_checkpoint": "/content/drive/MyDrive/hyperclova-deobfuscation-lora/checkpoint-1000",
+  "epoch": 2.487111111111111,
+  "eval_steps": 200,
+  "global_step": 1400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017777777777777778,
+      "grad_norm": 3.0164783000946045,
+      "learning_rate": 4.4999999999999996e-05,
+      "loss": 4.0476,
+      "mean_token_accuracy": 0.3606465607881546,
+      "num_tokens": 22106.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.035555555555555556,
+      "grad_norm": 2.489065408706665,
+      "learning_rate": 9.5e-05,
+      "loss": 3.3625,
+      "mean_token_accuracy": 0.46429387107491493,
+      "num_tokens": 44943.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 1.6735386848449707,
+      "learning_rate": 0.000145,
+      "loss": 2.9752,
+      "mean_token_accuracy": 0.5158864699304104,
+      "num_tokens": 67397.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.07111111111111111,
+      "grad_norm": 1.5050151348114014,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 2.7549,
+      "mean_token_accuracy": 0.5370393514633178,
+      "num_tokens": 89803.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.3750827312469482,
+      "learning_rate": 0.000245,
+      "loss": 2.6264,
+      "mean_token_accuracy": 0.550141765922308,
+      "num_tokens": 112364.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.10666666666666667,
+      "grad_norm": 1.4109346866607666,
+      "learning_rate": 0.000295,
+      "loss": 2.5264,
+      "mean_token_accuracy": 0.5618108093738556,
+      "num_tokens": 134028.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.12444444444444444,
+      "grad_norm": 1.3981140851974487,
+      "learning_rate": 0.000345,
+      "loss": 2.4839,
+      "mean_token_accuracy": 0.5678502731025219,
+      "num_tokens": 156703.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.14222222222222222,
+      "grad_norm": 1.4779222011566162,
+      "learning_rate": 0.000395,
+      "loss": 2.4262,
+      "mean_token_accuracy": 0.5784575901925564,
+      "num_tokens": 178986.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.1966180801391602,
+      "learning_rate": 0.00044500000000000003,
+      "loss": 2.4566,
+      "mean_token_accuracy": 0.5695327177643776,
+      "num_tokens": 201989.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.4000093936920166,
+      "learning_rate": 0.000495,
+      "loss": 2.3821,
+      "mean_token_accuracy": 0.5801047712564469,
+      "num_tokens": 223936.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.19555555555555557,
+      "grad_norm": 1.3838802576065063,
+      "learning_rate": 0.0004971680302076778,
+      "loss": 2.3444,
+      "mean_token_accuracy": 0.5860289096832275,
+      "num_tokens": 246101.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.3671435117721558,
+      "learning_rate": 0.0004940213971050976,
+      "loss": 2.3484,
+      "mean_token_accuracy": 0.5840608969330787,
+      "num_tokens": 269187.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.2311111111111111,
+      "grad_norm": 1.0929336547851562,
+      "learning_rate": 0.0004908747640025173,
+      "loss": 2.2781,
+      "mean_token_accuracy": 0.5930660709738731,
+      "num_tokens": 291454.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.24888888888888888,
+      "grad_norm": 1.1275103092193604,
+      "learning_rate": 0.00048772813089993704,
+      "loss": 2.2775,
+      "mean_token_accuracy": 0.5900422058999538,
+      "num_tokens": 314204.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1598752737045288,
+      "learning_rate": 0.0004845814977973568,
+      "loss": 2.242,
+      "mean_token_accuracy": 0.5971075922250748,
+      "num_tokens": 337167.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.28444444444444444,
+      "grad_norm": 1.2944793701171875,
+      "learning_rate": 0.0004814348646947766,
+      "loss": 2.2755,
+      "mean_token_accuracy": 0.5881656795740128,
+      "num_tokens": 360272.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.3022222222222222,
+      "grad_norm": 1.2451213598251343,
+      "learning_rate": 0.0004782882315921964,
+      "loss": 2.1925,
+      "mean_token_accuracy": 0.6079613700509071,
+      "num_tokens": 382779.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.168745517730713,
+      "learning_rate": 0.00047514159848961615,
+      "loss": 2.1997,
+      "mean_token_accuracy": 0.6048969209194184,
+      "num_tokens": 405438.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.3377777777777778,
+      "grad_norm": 1.2053375244140625,
+      "learning_rate": 0.0004719949653870359,
+      "loss": 2.236,
+      "mean_token_accuracy": 0.5962931856513023,
+      "num_tokens": 428660.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.1758276224136353,
+      "learning_rate": 0.00046884833228445566,
+      "loss": 2.1761,
+      "mean_token_accuracy": 0.6056372985243798,
+      "num_tokens": 450808.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "eval_loss": 2.166856527328491,
+      "eval_mean_token_accuracy": 0.6084163203239441,
+      "eval_num_tokens": 450808.0,
+      "eval_runtime": 30.2794,
+      "eval_samples_per_second": 33.026,
+      "eval_steps_per_second": 8.256,
+      "step": 200
+    },
+    {
+      "epoch": 0.37333333333333335,
+      "grad_norm": 1.3344407081604004,
+      "learning_rate": 0.0004657016991818754,
+      "loss": 2.1878,
+      "mean_token_accuracy": 0.6024597108364105,
+      "num_tokens": 473434.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.39111111111111113,
+      "grad_norm": 1.1835261583328247,
+      "learning_rate": 0.00046255506607929516,
+      "loss": 2.2188,
+      "mean_token_accuracy": 0.5974872022867203,
+      "num_tokens": 496482.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.4088888888888889,
+      "grad_norm": 1.1516607999801636,
+      "learning_rate": 0.00045940843297671494,
+      "loss": 2.1659,
+      "mean_token_accuracy": 0.604978509247303,
+      "num_tokens": 519379.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.1609504222869873,
+      "learning_rate": 0.00045626179987413466,
+      "loss": 2.1777,
+      "mean_token_accuracy": 0.6044537022709846,
+      "num_tokens": 542631.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.1110681295394897,
+      "learning_rate": 0.00045311516677155444,
+      "loss": 2.1604,
+      "mean_token_accuracy": 0.6043903902173042,
+      "num_tokens": 565400.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.4622222222222222,
+      "grad_norm": 1.2393710613250732,
+      "learning_rate": 0.0004499685336689742,
+      "loss": 2.0864,
+      "mean_token_accuracy": 0.6146551981568337,
+      "num_tokens": 588003.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.3670927286148071,
+      "learning_rate": 0.000446821900566394,
+      "loss": 2.1201,
+      "mean_token_accuracy": 0.6108790010213851,
+      "num_tokens": 610974.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.49777777777777776,
+      "grad_norm": 1.1501164436340332,
+      "learning_rate": 0.0004436752674638137,
+      "loss": 2.1437,
+      "mean_token_accuracy": 0.6097761034965515,
+      "num_tokens": 634093.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.5155555555555555,
+      "grad_norm": 1.1109156608581543,
+      "learning_rate": 0.00044052863436123345,
+      "loss": 2.0543,
+      "mean_token_accuracy": 0.6271390929818154,
+      "num_tokens": 656188.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.2343859672546387,
+      "learning_rate": 0.0004373820012586532,
+      "loss": 2.0971,
+      "mean_token_accuracy": 0.6169493526220322,
+      "num_tokens": 678769.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.5511111111111111,
+      "grad_norm": 1.083136796951294,
+      "learning_rate": 0.000434235368156073,
+      "loss": 2.0734,
+      "mean_token_accuracy": 0.6225788787007331,
+      "num_tokens": 701734.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.5688888888888889,
+      "grad_norm": 1.1497492790222168,
+      "learning_rate": 0.0004310887350534928,
+      "loss": 2.0603,
+      "mean_token_accuracy": 0.6220441117882729,
+      "num_tokens": 724815.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.1948379278182983,
+      "learning_rate": 0.00042794210195091256,
+      "loss": 2.0612,
+      "mean_token_accuracy": 0.6211912930011749,
+      "num_tokens": 746903.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.6044444444444445,
+      "grad_norm": 1.1575194597244263,
+      "learning_rate": 0.00042479546884833234,
+      "loss": 2.042,
+      "mean_token_accuracy": 0.6249620258808136,
+      "num_tokens": 768982.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.6222222222222222,
+      "grad_norm": 1.0737602710723877,
+      "learning_rate": 0.00042164883574575206,
+      "loss": 2.066,
+      "mean_token_accuracy": 0.6236828655004502,
+      "num_tokens": 791061.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.0655274391174316,
+      "learning_rate": 0.0004185022026431718,
+      "loss": 2.0556,
+      "mean_token_accuracy": 0.6238334342837334,
+      "num_tokens": 813112.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.6577777777777778,
+      "grad_norm": 1.2494139671325684,
+      "learning_rate": 0.00041535556954059156,
+      "loss": 2.0544,
+      "mean_token_accuracy": 0.623551732301712,
+      "num_tokens": 835469.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.6755555555555556,
+      "grad_norm": 1.251245379447937,
+      "learning_rate": 0.00041220893643801134,
+      "loss": 2.0623,
+      "mean_token_accuracy": 0.6137503892183304,
+      "num_tokens": 857795.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.6933333333333334,
+      "grad_norm": 1.1011890172958374,
+      "learning_rate": 0.0004090623033354311,
+      "loss": 2.0281,
+      "mean_token_accuracy": 0.6267385333776474,
+      "num_tokens": 879659.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 1.3007749319076538,
+      "learning_rate": 0.00040591567023285085,
+      "loss": 2.0135,
+      "mean_token_accuracy": 0.628263184428215,
+      "num_tokens": 902240.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "eval_loss": 2.0190296173095703,
+      "eval_mean_token_accuracy": 0.6285779621601105,
+      "eval_num_tokens": 902240.0,
+      "eval_runtime": 31.7727,
+      "eval_samples_per_second": 31.474,
+      "eval_steps_per_second": 7.868,
+      "step": 400
+    },
+    {
+      "epoch": 0.7288888888888889,
+      "grad_norm": 1.1896709203720093,
+      "learning_rate": 0.0004027690371302706,
+      "loss": 2.0396,
+      "mean_token_accuracy": 0.6259367674589157,
+      "num_tokens": 925091.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.3017927408218384,
+      "learning_rate": 0.0003996224040276904,
+      "loss": 2.0575,
+      "mean_token_accuracy": 0.6207560569047927,
+      "num_tokens": 948151.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.7644444444444445,
+      "grad_norm": 1.1772371530532837,
+      "learning_rate": 0.00039647577092511013,
+      "loss": 1.9884,
+      "mean_token_accuracy": 0.6323661148548126,
+      "num_tokens": 970339.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.7822222222222223,
+      "grad_norm": 1.0881459712982178,
+      "learning_rate": 0.0003933291378225299,
+      "loss": 2.0283,
+      "mean_token_accuracy": 0.6244203582406044,
+      "num_tokens": 993552.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1693931818008423,
+      "learning_rate": 0.00039018250471994963,
+      "loss": 2.0475,
+      "mean_token_accuracy": 0.6222739323973656,
+      "num_tokens": 1015564.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.8177777777777778,
+      "grad_norm": 1.2372338771820068,
+      "learning_rate": 0.0003870358716173694,
+      "loss": 1.9537,
+      "mean_token_accuracy": 0.6377866417169571,
+      "num_tokens": 1037721.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.8355555555555556,
+      "grad_norm": 1.1657993793487549,
+      "learning_rate": 0.0003838892385147892,
+      "loss": 2.0209,
+      "mean_token_accuracy": 0.6264078587293624,
+      "num_tokens": 1060627.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.1275551319122314,
+      "learning_rate": 0.00038074260541220897,
+      "loss": 1.9973,
+      "mean_token_accuracy": 0.6285641089081764,
+      "num_tokens": 1082672.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.8711111111111111,
+      "grad_norm": 1.162360429763794,
+      "learning_rate": 0.00037759597230962874,
+      "loss": 2.0117,
+      "mean_token_accuracy": 0.629520146548748,
+      "num_tokens": 1104857.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.1501044034957886,
+      "learning_rate": 0.00037444933920704847,
+      "loss": 1.9958,
+      "mean_token_accuracy": 0.6280175432562828,
+      "num_tokens": 1127612.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.1588389873504639,
+      "learning_rate": 0.0003713027061044682,
+      "loss": 2.0001,
+      "mean_token_accuracy": 0.6282740488648415,
+      "num_tokens": 1150411.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.9244444444444444,
+      "grad_norm": 1.1372511386871338,
+      "learning_rate": 0.00036815607300188797,
+      "loss": 2.0194,
+      "mean_token_accuracy": 0.6251062974333763,
+      "num_tokens": 1172683.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.9422222222222222,
+      "grad_norm": 1.1625277996063232,
+      "learning_rate": 0.00036500943989930775,
+      "loss": 1.9194,
+      "mean_token_accuracy": 0.6430759638547897,
+      "num_tokens": 1194160.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.2204004526138306,
+      "learning_rate": 0.00036186280679672753,
+      "loss": 1.9411,
+      "mean_token_accuracy": 0.6433150470256805,
+      "num_tokens": 1215760.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.9777777777777777,
+      "grad_norm": 1.1138572692871094,
+      "learning_rate": 0.00035871617369414725,
+      "loss": 2.0082,
+      "mean_token_accuracy": 0.6322961866855621,
+      "num_tokens": 1238191.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.9955555555555555,
+      "grad_norm": 1.1804752349853516,
+      "learning_rate": 0.00035556954059156703,
+      "loss": 1.9751,
+      "mean_token_accuracy": 0.6305457279086113,
+      "num_tokens": 1260429.0,
+      "step": 560
+    },
+    {
+      "epoch": 1.0124444444444445,
+      "grad_norm": 1.0810185670852661,
+      "learning_rate": 0.0003524229074889868,
+      "loss": 1.8437,
+      "mean_token_accuracy": 0.6523516507525193,
+      "num_tokens": 1281232.0,
+      "step": 570
+    },
+    {
+      "epoch": 1.0302222222222222,
+      "grad_norm": 1.2796133756637573,
+      "learning_rate": 0.0003492762743864066,
+      "loss": 1.7348,
+      "mean_token_accuracy": 0.6675047099590301,
+      "num_tokens": 1304130.0,
+      "step": 580
+    },
+    {
+      "epoch": 1.048,
+      "grad_norm": 1.1196218729019165,
+      "learning_rate": 0.0003461296412838263,
+      "loss": 1.7244,
+      "mean_token_accuracy": 0.6726136475801467,
+      "num_tokens": 1326753.0,
+      "step": 590
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "grad_norm": 1.1148934364318848,
+      "learning_rate": 0.00034298300818124604,
+      "loss": 1.7216,
+      "mean_token_accuracy": 0.6700241908431053,
+      "num_tokens": 1348985.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "eval_loss": 1.9544994831085205,
+      "eval_mean_token_accuracy": 0.638362931728363,
+      "eval_num_tokens": 1348985.0,
+      "eval_runtime": 30.1088,
+      "eval_samples_per_second": 33.213,
+      "eval_steps_per_second": 8.303,
+      "step": 600
+    },
+    {
+      "epoch": 1.0835555555555556,
+      "grad_norm": 1.0550259351730347,
+      "learning_rate": 0.0003398363750786658,
+      "loss": 1.734,
+      "mean_token_accuracy": 0.670471416413784,
+      "num_tokens": 1371318.0,
+      "step": 610
+    },
+    {
+      "epoch": 1.1013333333333333,
+      "grad_norm": 1.1968863010406494,
+      "learning_rate": 0.0003366897419760856,
+      "loss": 1.7819,
+      "mean_token_accuracy": 0.661465460062027,
+      "num_tokens": 1393816.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.1191111111111112,
+      "grad_norm": 1.2184885740280151,
+      "learning_rate": 0.0003335431088735054,
+      "loss": 1.769,
+      "mean_token_accuracy": 0.6628920555114746,
+      "num_tokens": 1416529.0,
+      "step": 630
+    },
+    {
+      "epoch": 1.1368888888888888,
+      "grad_norm": 1.2553062438964844,
+      "learning_rate": 0.00033039647577092515,
+      "loss": 1.7807,
+      "mean_token_accuracy": 0.6629297524690628,
+      "num_tokens": 1438970.0,
+      "step": 640
+    },
+    {
+      "epoch": 1.1546666666666667,
+      "grad_norm": 1.1857445240020752,
+      "learning_rate": 0.0003272498426683449,
+      "loss": 1.7555,
+      "mean_token_accuracy": 0.6634594619274139,
+      "num_tokens": 1461324.0,
+      "step": 650
+    },
+    {
+      "epoch": 1.1724444444444444,
+      "grad_norm": 1.2381750345230103,
+      "learning_rate": 0.0003241032095657646,
+      "loss": 1.7925,
+      "mean_token_accuracy": 0.6549596875905991,
+      "num_tokens": 1484246.0,
+      "step": 660
+    },
+    {
+      "epoch": 1.1902222222222223,
+      "grad_norm": 1.187692642211914,
+      "learning_rate": 0.0003209565764631844,
+      "loss": 1.7776,
+      "mean_token_accuracy": 0.658722423017025,
+      "num_tokens": 1507305.0,
+      "step": 670
+    },
+    {
+      "epoch": 1.208,
+      "grad_norm": 1.1748046875,
+      "learning_rate": 0.00031780994336060416,
+      "loss": 1.7355,
+      "mean_token_accuracy": 0.6706653699278832,
+      "num_tokens": 1529082.0,
+      "step": 680
+    },
+    {
+      "epoch": 1.2257777777777779,
+      "grad_norm": 1.2238432168960571,
+      "learning_rate": 0.00031466331025802394,
+      "loss": 1.7462,
+      "mean_token_accuracy": 0.6667303949594497,
+      "num_tokens": 1551159.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.2435555555555555,
+      "grad_norm": 1.1904925107955933,
+      "learning_rate": 0.00031151667715544366,
+      "loss": 1.7627,
+      "mean_token_accuracy": 0.6670840814709663,
+      "num_tokens": 1573348.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.2613333333333334,
+      "grad_norm": 1.282875895500183,
+      "learning_rate": 0.00030837004405286344,
+      "loss": 1.7625,
+      "mean_token_accuracy": 0.6660559684038162,
+      "num_tokens": 1595989.0,
+      "step": 710
+    },
+    {
+      "epoch": 1.279111111111111,
+      "grad_norm": 1.2079155445098877,
+      "learning_rate": 0.0003052234109502832,
+      "loss": 1.7979,
+      "mean_token_accuracy": 0.6636087343096733,
+      "num_tokens": 1617895.0,
+      "step": 720
+    },
+    {
+      "epoch": 1.2968888888888888,
+      "grad_norm": 1.1903313398361206,
+      "learning_rate": 0.000302076777847703,
+      "loss": 1.7552,
+      "mean_token_accuracy": 0.6641585499048233,
+      "num_tokens": 1640415.0,
+      "step": 730
+    },
+    {
+      "epoch": 1.3146666666666667,
+      "grad_norm": 1.1872097253799438,
+      "learning_rate": 0.0002989301447451227,
+      "loss": 1.74,
+      "mean_token_accuracy": 0.6656632766127586,
+      "num_tokens": 1662713.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.3324444444444445,
+      "grad_norm": 1.1994388103485107,
+      "learning_rate": 0.00029578351164254244,
+      "loss": 1.7757,
+      "mean_token_accuracy": 0.6637446776032447,
+      "num_tokens": 1684783.0,
+      "step": 750
+    },
+    {
+      "epoch": 1.3502222222222222,
+      "grad_norm": 1.2072243690490723,
+      "learning_rate": 0.0002926368785399622,
+      "loss": 1.7702,
+      "mean_token_accuracy": 0.6637947261333466,
+      "num_tokens": 1706623.0,
+      "step": 760
+    },
+    {
+      "epoch": 1.3679999999999999,
+      "grad_norm": 1.2028874158859253,
+      "learning_rate": 0.000289490245437382,
+      "loss": 1.7952,
+      "mean_token_accuracy": 0.6561207175254822,
+      "num_tokens": 1729494.0,
+      "step": 770
+    },
+    {
+      "epoch": 1.3857777777777778,
+      "grad_norm": 1.2276169061660767,
+      "learning_rate": 0.0002863436123348018,
+      "loss": 1.7624,
+      "mean_token_accuracy": 0.664546848833561,
+      "num_tokens": 1752045.0,
+      "step": 780
+    },
+    {
+      "epoch": 1.4035555555555557,
+      "grad_norm": 1.1520951986312866,
+      "learning_rate": 0.00028319697923222156,
+      "loss": 1.7743,
+      "mean_token_accuracy": 0.6641672790050507,
+      "num_tokens": 1775601.0,
+      "step": 790
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "grad_norm": 1.219901204109192,
+      "learning_rate": 0.0002800503461296413,
+      "loss": 1.7346,
+      "mean_token_accuracy": 0.6647952407598495,
+      "num_tokens": 1798633.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "eval_loss": 1.9113014936447144,
+      "eval_mean_token_accuracy": 0.6443024153709411,
+      "eval_num_tokens": 1798633.0,
+      "eval_runtime": 30.1829,
+      "eval_samples_per_second": 33.131,
+      "eval_steps_per_second": 8.283,
+      "step": 800
+    },
+    {
+      "epoch": 1.439111111111111,
+      "grad_norm": 1.2413419485092163,
+      "learning_rate": 0.00027690371302706106,
+      "loss": 1.7157,
+      "mean_token_accuracy": 0.6692047983407974,
+      "num_tokens": 1820656.0,
+      "step": 810
+    },
+    {
+      "epoch": 1.456888888888889,
+      "grad_norm": 1.2917033433914185,
+      "learning_rate": 0.0002737570799244808,
+      "loss": 1.7631,
+      "mean_token_accuracy": 0.6633412629365921,
+      "num_tokens": 1842871.0,
+      "step": 820
+    },
+    {
+      "epoch": 1.4746666666666668,
+      "grad_norm": 1.2017630338668823,
+      "learning_rate": 0.00027061044682190056,
+      "loss": 1.726,
+      "mean_token_accuracy": 0.6701689288020134,
+      "num_tokens": 1865652.0,
+      "step": 830
+    },
+    {
+      "epoch": 1.4924444444444445,
+      "grad_norm": 1.1332948207855225,
+      "learning_rate": 0.00026746381371932034,
+      "loss": 1.7561,
+      "mean_token_accuracy": 0.6648476481437683,
+      "num_tokens": 1888455.0,
+      "step": 840
+    },
+    {
+      "epoch": 1.5102222222222221,
+      "grad_norm": 1.2592897415161133,
+      "learning_rate": 0.00026431718061674007,
+      "loss": 1.7194,
+      "mean_token_accuracy": 0.6733420416712761,
+      "num_tokens": 1910676.0,
+      "step": 850
+    },
+    {
+      "epoch": 1.528,
+      "grad_norm": 1.2740918397903442,
+      "learning_rate": 0.00026117054751415985,
+      "loss": 1.7502,
+      "mean_token_accuracy": 0.6677413523197174,
+      "num_tokens": 1933253.0,
+      "step": 860
+    },
+    {
+      "epoch": 1.545777777777778,
+      "grad_norm": 1.2723543643951416,
+      "learning_rate": 0.0002580239144115796,
+      "loss": 1.7622,
+      "mean_token_accuracy": 0.6622136697173119,
+      "num_tokens": 1955820.0,
+      "step": 870
+    },
+    {
+      "epoch": 1.5635555555555556,
+      "grad_norm": 1.2220773696899414,
+      "learning_rate": 0.0002548772813089994,
+      "loss": 1.7568,
+      "mean_token_accuracy": 0.6640823170542717,
+      "num_tokens": 1978120.0,
+      "step": 880
+    },
+    {
+      "epoch": 1.5813333333333333,
+      "grad_norm": 1.2161861658096313,
+      "learning_rate": 0.0002517306482064191,
+      "loss": 1.7031,
+      "mean_token_accuracy": 0.6733875796198845,
+      "num_tokens": 2000508.0,
+      "step": 890
+    },
+    {
+      "epoch": 1.5991111111111111,
+      "grad_norm": 1.1472899913787842,
+      "learning_rate": 0.0002485840151038389,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6644304901361465,
+      "num_tokens": 2023356.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.616888888888889,
+      "grad_norm": 1.177524447441101,
+      "learning_rate": 0.00024543738200125863,
+      "loss": 1.6951,
+      "mean_token_accuracy": 0.6781735569238663,
+      "num_tokens": 2045965.0,
+      "step": 910
+    },
+    {
+      "epoch": 1.6346666666666667,
+      "grad_norm": 1.158720850944519,
+      "learning_rate": 0.0002422907488986784,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6616655588150024,
+      "num_tokens": 2069108.0,
+      "step": 920
+    },
+    {
+      "epoch": 1.6524444444444444,
+      "grad_norm": 1.3048863410949707,
+      "learning_rate": 0.0002391441157960982,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6625866696238518,
+      "num_tokens": 2091755.0,
+      "step": 930
+    },
+    {
+      "epoch": 1.6702222222222223,
+      "grad_norm": 1.3040592670440674,
+      "learning_rate": 0.00023599748269351794,
+      "loss": 1.7286,
+      "mean_token_accuracy": 0.6688349068164825,
+      "num_tokens": 2114496.0,
+      "step": 940
+    },
+    {
+      "epoch": 1.688,
+      "grad_norm": 1.1945664882659912,
+      "learning_rate": 0.0002328508495909377,
+      "loss": 1.7479,
+      "mean_token_accuracy": 0.6634002208709717,
+      "num_tokens": 2137041.0,
+      "step": 950
+    },
+    {
+      "epoch": 1.7057777777777776,
+      "grad_norm": 1.2552154064178467,
+      "learning_rate": 0.00022970421648835747,
+      "loss": 1.7537,
+      "mean_token_accuracy": 0.6634084343910217,
+      "num_tokens": 2159711.0,
+      "step": 960
+    },
+    {
+      "epoch": 1.7235555555555555,
+      "grad_norm": 1.2484403848648071,
+      "learning_rate": 0.00022655758338577722,
+      "loss": 1.7557,
+      "mean_token_accuracy": 0.6687701687216758,
+      "num_tokens": 2181675.0,
+      "step": 970
+    },
+    {
+      "epoch": 1.7413333333333334,
+      "grad_norm": 1.1729741096496582,
+      "learning_rate": 0.000223410950283197,
+      "loss": 1.7274,
+      "mean_token_accuracy": 0.6730923235416413,
+      "num_tokens": 2204671.0,
+      "step": 980
+    },
+    {
+      "epoch": 1.759111111111111,
+      "grad_norm": 1.312406301498413,
+      "learning_rate": 0.00022026431718061672,
+      "loss": 1.7377,
+      "mean_token_accuracy": 0.6612286046147346,
+      "num_tokens": 2227866.0,
+      "step": 990
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "grad_norm": 1.3160700798034668,
+      "learning_rate": 0.0002171176840780365,
+      "loss": 1.7224,
+      "mean_token_accuracy": 0.6710903704166412,
+      "num_tokens": 2250379.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "eval_loss": 1.8698750734329224,
+      "eval_mean_token_accuracy": 0.6504300956726075,
+      "eval_num_tokens": 2250379.0,
+      "eval_runtime": 30.1651,
+      "eval_samples_per_second": 33.151,
+      "eval_steps_per_second": 8.288,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7946666666666666,
+      "grad_norm": 1.1682268381118774,
+      "learning_rate": 0.00021397105097545628,
+      "loss": 1.7458,
+      "mean_token_accuracy": 0.665731918811798,
+      "num_tokens": 2272999.0,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8124444444444445,
+      "grad_norm": 1.1300792694091797,
+      "learning_rate": 0.00021082441787287603,
+      "loss": 1.7261,
+      "mean_token_accuracy": 0.6688766568899155,
+      "num_tokens": 2295110.0,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8302222222222222,
+      "grad_norm": 1.2450717687606812,
+      "learning_rate": 0.00020767778477029578,
+      "loss": 1.6968,
+      "mean_token_accuracy": 0.6718593999743462,
+      "num_tokens": 2317432.0,
+      "step": 1030
+    },
+    {
+      "epoch": 1.8479999999999999,
+      "grad_norm": 1.201785683631897,
+      "learning_rate": 0.00020453115166771556,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6636266440153122,
+      "num_tokens": 2340305.0,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8657777777777778,
+      "grad_norm": 1.2161189317703247,
+      "learning_rate": 0.0002013845185651353,
+      "loss": 1.7335,
+      "mean_token_accuracy": 0.6697820112109184,
+      "num_tokens": 2362746.0,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8835555555555556,
+      "grad_norm": 1.209251046180725,
+      "learning_rate": 0.00019823788546255506,
+      "loss": 1.7197,
+      "mean_token_accuracy": 0.6663407534360886,
+      "num_tokens": 2385647.0,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9013333333333333,
+      "grad_norm": 1.1715242862701416,
+      "learning_rate": 0.00019509125235997482,
+      "loss": 1.7298,
+      "mean_token_accuracy": 0.6650380924344063,
+      "num_tokens": 2408732.0,
+      "step": 1070
+    },
+    {
+      "epoch": 1.919111111111111,
+      "grad_norm": 1.2717958688735962,
+      "learning_rate": 0.0001919446192573946,
+      "loss": 1.7101,
+      "mean_token_accuracy": 0.672779667377472,
+      "num_tokens": 2431137.0,
+      "step": 1080
+    },
+    {
+      "epoch": 1.9368888888888889,
+      "grad_norm": 1.2264835834503174,
+      "learning_rate": 0.00018879798615481437,
+      "loss": 1.6754,
+      "mean_token_accuracy": 0.6768045455217362,
+      "num_tokens": 2453517.0,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9546666666666668,
+      "grad_norm": 1.3289543390274048,
+      "learning_rate": 0.0001856513530522341,
+      "loss": 1.7178,
+      "mean_token_accuracy": 0.6685071781277656,
+      "num_tokens": 2475737.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9724444444444444,
+      "grad_norm": 1.1367521286010742,
+      "learning_rate": 0.00018250471994965388,
+      "loss": 1.7109,
+      "mean_token_accuracy": 0.6677573248744011,
+      "num_tokens": 2497880.0,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9902222222222221,
+      "grad_norm": 1.246870994567871,
+      "learning_rate": 0.00017935808684707363,
+      "loss": 1.7317,
+      "mean_token_accuracy": 0.6695448830723763,
+      "num_tokens": 2519974.0,
+      "step": 1120
+    },
+    {
+      "epoch": 2.007111111111111,
+      "grad_norm": 1.0757529735565186,
+      "learning_rate": 0.0001762114537444934,
+      "loss": 1.586,
+      "mean_token_accuracy": 0.6983135671992051,
+      "num_tokens": 2540296.0,
+      "step": 1130
+    },
+    {
+      "epoch": 2.024888888888889,
+      "grad_norm": 1.3501553535461426,
+      "learning_rate": 0.00017306482064191316,
+      "loss": 1.4761,
+      "mean_token_accuracy": 0.7080946862697601,
+      "num_tokens": 2561919.0,
+      "step": 1140
+    },
+    {
+      "epoch": 2.042666666666667,
+      "grad_norm": 1.502793788909912,
+      "learning_rate": 0.0001699181875393329,
+      "loss": 1.5049,
+      "mean_token_accuracy": 0.7034129843115806,
+      "num_tokens": 2584915.0,
+      "step": 1150
+    },
+    {
+      "epoch": 2.0604444444444443,
+      "grad_norm": 1.2668856382369995,
+      "learning_rate": 0.0001667715544367527,
+      "loss": 1.4612,
+      "mean_token_accuracy": 0.7100765943527222,
+      "num_tokens": 2606857.0,
+      "step": 1160
+    },
+    {
+      "epoch": 2.078222222222222,
+      "grad_norm": 1.386775016784668,
+      "learning_rate": 0.00016362492133417244,
+      "loss": 1.4695,
+      "mean_token_accuracy": 0.7058908969163895,
+      "num_tokens": 2629737.0,
+      "step": 1170
+    },
+    {
+      "epoch": 2.096,
+      "grad_norm": 1.3320142030715942,
+      "learning_rate": 0.0001604782882315922,
+      "loss": 1.4337,
+      "mean_token_accuracy": 0.7136739388108253,
+      "num_tokens": 2651936.0,
+      "step": 1180
+    },
+    {
+      "epoch": 2.113777777777778,
+      "grad_norm": 1.289437174797058,
+      "learning_rate": 0.00015733165512901197,
+      "loss": 1.4975,
+      "mean_token_accuracy": 0.7037445783615113,
+      "num_tokens": 2675241.0,
+      "step": 1190
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "grad_norm": 1.208194613456726,
+      "learning_rate": 0.00015418502202643172,
+      "loss": 1.4769,
+      "mean_token_accuracy": 0.7093130722641945,
+      "num_tokens": 2698219.0,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "eval_loss": 1.8875290155410767,
+      "eval_mean_token_accuracy": 0.6501186113357544,
+      "eval_num_tokens": 2698219.0,
+      "eval_runtime": 30.175,
+      "eval_samples_per_second": 33.14,
+      "eval_steps_per_second": 8.285,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1493333333333333,
+      "grad_norm": 1.360378623008728,
+      "learning_rate": 0.0001510383889238515,
+      "loss": 1.485,
+      "mean_token_accuracy": 0.7059277340769767,
+      "num_tokens": 2721580.0,
+      "step": 1210
+    },
+    {
+      "epoch": 2.167111111111111,
+      "grad_norm": 1.3289263248443604,
+      "learning_rate": 0.00014789175582127122,
+      "loss": 1.4869,
+      "mean_token_accuracy": 0.7061908945441246,
+      "num_tokens": 2744110.0,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1848888888888887,
+      "grad_norm": 1.2693774700164795,
+      "learning_rate": 0.000144745122718691,
+      "loss": 1.4555,
+      "mean_token_accuracy": 0.712950111925602,
+      "num_tokens": 2765844.0,
+      "step": 1230
+    },
+    {
+      "epoch": 2.2026666666666666,
+      "grad_norm": 1.3220160007476807,
+      "learning_rate": 0.00014159848961611078,
+      "loss": 1.4634,
+      "mean_token_accuracy": 0.7090483531355858,
+      "num_tokens": 2788070.0,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2204444444444444,
+      "grad_norm": 1.2682182788848877,
+      "learning_rate": 0.00013845185651353053,
+      "loss": 1.497,
+      "mean_token_accuracy": 0.7039189741015435,
+      "num_tokens": 2810964.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.2382222222222223,
+      "grad_norm": 1.3531242609024048,
+      "learning_rate": 0.00013530522341095028,
+      "loss": 1.4625,
+      "mean_token_accuracy": 0.7105821236968041,
+      "num_tokens": 2833176.0,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.2819761037826538,
+      "learning_rate": 0.00013215859030837003,
+      "loss": 1.5092,
+      "mean_token_accuracy": 0.7021819025278091,
+      "num_tokens": 2855440.0,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2737777777777777,
+      "grad_norm": 1.3131153583526611,
+      "learning_rate": 0.0001290119572057898,
+      "loss": 1.4458,
+      "mean_token_accuracy": 0.7121976673603058,
+      "num_tokens": 2877713.0,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2915555555555556,
+      "grad_norm": 1.2332444190979004,
+      "learning_rate": 0.00012586532410320956,
+      "loss": 1.4797,
+      "mean_token_accuracy": 0.7103414624929428,
+      "num_tokens": 2899934.0,
+      "step": 1290
+    },
+    {
+      "epoch": 2.3093333333333335,
+      "grad_norm": 1.2947789430618286,
+      "learning_rate": 0.00012271869100062931,
+      "loss": 1.4969,
+      "mean_token_accuracy": 0.7032593175768852,
+      "num_tokens": 2923244.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.327111111111111,
+      "grad_norm": 1.4002538919448853,
+      "learning_rate": 0.0001195720578980491,
+      "loss": 1.4538,
+      "mean_token_accuracy": 0.711962579190731,
+      "num_tokens": 2945580.0,
+      "step": 1310
+    },
+    {
+      "epoch": 2.344888888888889,
+      "grad_norm": 1.270635962486267,
+      "learning_rate": 0.00011642542479546884,
+      "loss": 1.4622,
+      "mean_token_accuracy": 0.7059817537665367,
+      "num_tokens": 2969470.0,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3626666666666667,
+      "grad_norm": 1.3131049871444702,
+      "learning_rate": 0.00011327879169288861,
+      "loss": 1.4597,
+      "mean_token_accuracy": 0.7036674708127976,
+      "num_tokens": 2991982.0,
+      "step": 1330
+    },
+    {
+      "epoch": 2.3804444444444446,
+      "grad_norm": 1.3708815574645996,
+      "learning_rate": 0.00011013215859030836,
+      "loss": 1.4837,
+      "mean_token_accuracy": 0.706661482155323,
+      "num_tokens": 3014206.0,
+      "step": 1340
+    },
+    {
+      "epoch": 2.398222222222222,
+      "grad_norm": 1.3198243379592896,
+      "learning_rate": 0.00010698552548772814,
+      "loss": 1.4431,
+      "mean_token_accuracy": 0.7101009368896485,
+      "num_tokens": 3036651.0,
+      "step": 1350
+    },
+    {
+      "epoch": 2.416,
+      "grad_norm": 1.280879259109497,
+      "learning_rate": 0.00010383889238514789,
+      "loss": 1.477,
+      "mean_token_accuracy": 0.7085638150572777,
+      "num_tokens": 3058745.0,
+      "step": 1360
+    },
+    {
+      "epoch": 2.433777777777778,
+      "grad_norm": 1.419290542602539,
+      "learning_rate": 0.00010069225928256766,
+      "loss": 1.4887,
+      "mean_token_accuracy": 0.7048677980899811,
+      "num_tokens": 3081726.0,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4515555555555557,
+      "grad_norm": 1.3684442043304443,
+      "learning_rate": 9.754562617998741e-05,
+      "loss": 1.4677,
+      "mean_token_accuracy": 0.7087965488433838,
+      "num_tokens": 3104292.0,
+      "step": 1380
+    },
+    {
+      "epoch": 2.469333333333333,
+      "grad_norm": 1.3453881740570068,
+      "learning_rate": 9.439899307740719e-05,
+      "loss": 1.4682,
+      "mean_token_accuracy": 0.7076600238680839,
+      "num_tokens": 3126334.0,
+      "step": 1390
+    },
+    {
+      "epoch": 2.487111111111111,
+      "grad_norm": 1.2759828567504883,
+      "learning_rate": 9.125235997482694e-05,
+      "loss": 1.4971,
+      "mean_token_accuracy": 0.7039492294192314,
+      "num_tokens": 3149845.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.487111111111111,
+      "eval_loss": 1.873150110244751,
+      "eval_mean_token_accuracy": 0.65266099858284,
+      "eval_num_tokens": 3149845.0,
+      "eval_runtime": 30.0464,
+      "eval_samples_per_second": 33.282,
+      "eval_steps_per_second": 8.32,
+      "step": 1400
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1689,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1090406995533824e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd15c2ce737a7ea199ddf9399d7f90d562a3ce4f7e1a07c04ec4267cbad52d6d
+size 5624

checkpoint-1600/README.md

@@ -0,0 +1,202 @@
+---
+base_model: naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

checkpoint-1600/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-1600/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a6551b8c58ce9d8961031fb6846b0764b6df04c21e7a050afc85c235ceb7be
+size 39366152

checkpoint-1600/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

checkpoint-1600/chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1600/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:214394ab6cd77edd3960c377228f254205205119eb0992bb3e03edf84c64ccb7
+size 78925378

checkpoint-1600/special_tokens_map.json

@@ -0,0 +1,86 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endofturn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1600/tokenizer_config.json

@@ -0,0 +1,501 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|_unuse_missing_100256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|_unuse_missing_100261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|_unuse_missing_100262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|_unuse_missing_100263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|_unuse_missing_100264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|_unuse_missing_100265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|_unuse_missing_100266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|_unuse_missing_100267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|_unuse_missing_100268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|_unuse_missing_100269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|_unuse_missing_100270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|_unuse_missing_100271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|stop|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|endofturn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110491": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110492": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110493": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110494": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110495": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110496": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110497": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110498": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110499": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110500": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110501": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110502": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110503": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110504": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110505": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110506": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110507": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110508": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110509": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110510": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110511": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110512": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110513": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110514": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110515": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110516": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110517": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110518": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110519": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110520": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110521": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110522": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110523": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endofturn|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1600/trainer_state.json

@@ -0,0 +1,1554 @@
+{
+  "best_global_step": 1600,
+  "best_metric": 1.8612734079360962,
+  "best_model_checkpoint": "/content/drive/MyDrive/hyperclova-deobfuscation-lora/checkpoint-1600",
+  "epoch": 2.8426666666666667,
+  "eval_steps": 200,
+  "global_step": 1600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017777777777777778,
+      "grad_norm": 3.0164783000946045,
+      "learning_rate": 4.4999999999999996e-05,
+      "loss": 4.0476,
+      "mean_token_accuracy": 0.3606465607881546,
+      "num_tokens": 22106.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.035555555555555556,
+      "grad_norm": 2.489065408706665,
+      "learning_rate": 9.5e-05,
+      "loss": 3.3625,
+      "mean_token_accuracy": 0.46429387107491493,
+      "num_tokens": 44943.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 1.6735386848449707,
+      "learning_rate": 0.000145,
+      "loss": 2.9752,
+      "mean_token_accuracy": 0.5158864699304104,
+      "num_tokens": 67397.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.07111111111111111,
+      "grad_norm": 1.5050151348114014,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 2.7549,
+      "mean_token_accuracy": 0.5370393514633178,
+      "num_tokens": 89803.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.3750827312469482,
+      "learning_rate": 0.000245,
+      "loss": 2.6264,
+      "mean_token_accuracy": 0.550141765922308,
+      "num_tokens": 112364.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.10666666666666667,
+      "grad_norm": 1.4109346866607666,
+      "learning_rate": 0.000295,
+      "loss": 2.5264,
+      "mean_token_accuracy": 0.5618108093738556,
+      "num_tokens": 134028.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.12444444444444444,
+      "grad_norm": 1.3981140851974487,
+      "learning_rate": 0.000345,
+      "loss": 2.4839,
+      "mean_token_accuracy": 0.5678502731025219,
+      "num_tokens": 156703.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.14222222222222222,
+      "grad_norm": 1.4779222011566162,
+      "learning_rate": 0.000395,
+      "loss": 2.4262,
+      "mean_token_accuracy": 0.5784575901925564,
+      "num_tokens": 178986.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.1966180801391602,
+      "learning_rate": 0.00044500000000000003,
+      "loss": 2.4566,
+      "mean_token_accuracy": 0.5695327177643776,
+      "num_tokens": 201989.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.4000093936920166,
+      "learning_rate": 0.000495,
+      "loss": 2.3821,
+      "mean_token_accuracy": 0.5801047712564469,
+      "num_tokens": 223936.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.19555555555555557,
+      "grad_norm": 1.3838802576065063,
+      "learning_rate": 0.0004971680302076778,
+      "loss": 2.3444,
+      "mean_token_accuracy": 0.5860289096832275,
+      "num_tokens": 246101.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.3671435117721558,
+      "learning_rate": 0.0004940213971050976,
+      "loss": 2.3484,
+      "mean_token_accuracy": 0.5840608969330787,
+      "num_tokens": 269187.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.2311111111111111,
+      "grad_norm": 1.0929336547851562,
+      "learning_rate": 0.0004908747640025173,
+      "loss": 2.2781,
+      "mean_token_accuracy": 0.5930660709738731,
+      "num_tokens": 291454.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.24888888888888888,
+      "grad_norm": 1.1275103092193604,
+      "learning_rate": 0.00048772813089993704,
+      "loss": 2.2775,
+      "mean_token_accuracy": 0.5900422058999538,
+      "num_tokens": 314204.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1598752737045288,
+      "learning_rate": 0.0004845814977973568,
+      "loss": 2.242,
+      "mean_token_accuracy": 0.5971075922250748,
+      "num_tokens": 337167.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.28444444444444444,
+      "grad_norm": 1.2944793701171875,
+      "learning_rate": 0.0004814348646947766,
+      "loss": 2.2755,
+      "mean_token_accuracy": 0.5881656795740128,
+      "num_tokens": 360272.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.3022222222222222,
+      "grad_norm": 1.2451213598251343,
+      "learning_rate": 0.0004782882315921964,
+      "loss": 2.1925,
+      "mean_token_accuracy": 0.6079613700509071,
+      "num_tokens": 382779.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.168745517730713,
+      "learning_rate": 0.00047514159848961615,
+      "loss": 2.1997,
+      "mean_token_accuracy": 0.6048969209194184,
+      "num_tokens": 405438.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.3377777777777778,
+      "grad_norm": 1.2053375244140625,
+      "learning_rate": 0.0004719949653870359,
+      "loss": 2.236,
+      "mean_token_accuracy": 0.5962931856513023,
+      "num_tokens": 428660.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.1758276224136353,
+      "learning_rate": 0.00046884833228445566,
+      "loss": 2.1761,
+      "mean_token_accuracy": 0.6056372985243798,
+      "num_tokens": 450808.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "eval_loss": 2.166856527328491,
+      "eval_mean_token_accuracy": 0.6084163203239441,
+      "eval_num_tokens": 450808.0,
+      "eval_runtime": 30.2794,
+      "eval_samples_per_second": 33.026,
+      "eval_steps_per_second": 8.256,
+      "step": 200
+    },
+    {
+      "epoch": 0.37333333333333335,
+      "grad_norm": 1.3344407081604004,
+      "learning_rate": 0.0004657016991818754,
+      "loss": 2.1878,
+      "mean_token_accuracy": 0.6024597108364105,
+      "num_tokens": 473434.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.39111111111111113,
+      "grad_norm": 1.1835261583328247,
+      "learning_rate": 0.00046255506607929516,
+      "loss": 2.2188,
+      "mean_token_accuracy": 0.5974872022867203,
+      "num_tokens": 496482.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.4088888888888889,
+      "grad_norm": 1.1516607999801636,
+      "learning_rate": 0.00045940843297671494,
+      "loss": 2.1659,
+      "mean_token_accuracy": 0.604978509247303,
+      "num_tokens": 519379.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.1609504222869873,
+      "learning_rate": 0.00045626179987413466,
+      "loss": 2.1777,
+      "mean_token_accuracy": 0.6044537022709846,
+      "num_tokens": 542631.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.1110681295394897,
+      "learning_rate": 0.00045311516677155444,
+      "loss": 2.1604,
+      "mean_token_accuracy": 0.6043903902173042,
+      "num_tokens": 565400.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.4622222222222222,
+      "grad_norm": 1.2393710613250732,
+      "learning_rate": 0.0004499685336689742,
+      "loss": 2.0864,
+      "mean_token_accuracy": 0.6146551981568337,
+      "num_tokens": 588003.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.3670927286148071,
+      "learning_rate": 0.000446821900566394,
+      "loss": 2.1201,
+      "mean_token_accuracy": 0.6108790010213851,
+      "num_tokens": 610974.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.49777777777777776,
+      "grad_norm": 1.1501164436340332,
+      "learning_rate": 0.0004436752674638137,
+      "loss": 2.1437,
+      "mean_token_accuracy": 0.6097761034965515,
+      "num_tokens": 634093.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.5155555555555555,
+      "grad_norm": 1.1109156608581543,
+      "learning_rate": 0.00044052863436123345,
+      "loss": 2.0543,
+      "mean_token_accuracy": 0.6271390929818154,
+      "num_tokens": 656188.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.2343859672546387,
+      "learning_rate": 0.0004373820012586532,
+      "loss": 2.0971,
+      "mean_token_accuracy": 0.6169493526220322,
+      "num_tokens": 678769.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.5511111111111111,
+      "grad_norm": 1.083136796951294,
+      "learning_rate": 0.000434235368156073,
+      "loss": 2.0734,
+      "mean_token_accuracy": 0.6225788787007331,
+      "num_tokens": 701734.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.5688888888888889,
+      "grad_norm": 1.1497492790222168,
+      "learning_rate": 0.0004310887350534928,
+      "loss": 2.0603,
+      "mean_token_accuracy": 0.6220441117882729,
+      "num_tokens": 724815.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.1948379278182983,
+      "learning_rate": 0.00042794210195091256,
+      "loss": 2.0612,
+      "mean_token_accuracy": 0.6211912930011749,
+      "num_tokens": 746903.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.6044444444444445,
+      "grad_norm": 1.1575194597244263,
+      "learning_rate": 0.00042479546884833234,
+      "loss": 2.042,
+      "mean_token_accuracy": 0.6249620258808136,
+      "num_tokens": 768982.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.6222222222222222,
+      "grad_norm": 1.0737602710723877,
+      "learning_rate": 0.00042164883574575206,
+      "loss": 2.066,
+      "mean_token_accuracy": 0.6236828655004502,
+      "num_tokens": 791061.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.0655274391174316,
+      "learning_rate": 0.0004185022026431718,
+      "loss": 2.0556,
+      "mean_token_accuracy": 0.6238334342837334,
+      "num_tokens": 813112.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.6577777777777778,
+      "grad_norm": 1.2494139671325684,
+      "learning_rate": 0.00041535556954059156,
+      "loss": 2.0544,
+      "mean_token_accuracy": 0.623551732301712,
+      "num_tokens": 835469.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.6755555555555556,
+      "grad_norm": 1.251245379447937,
+      "learning_rate": 0.00041220893643801134,
+      "loss": 2.0623,
+      "mean_token_accuracy": 0.6137503892183304,
+      "num_tokens": 857795.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.6933333333333334,
+      "grad_norm": 1.1011890172958374,
+      "learning_rate": 0.0004090623033354311,
+      "loss": 2.0281,
+      "mean_token_accuracy": 0.6267385333776474,
+      "num_tokens": 879659.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 1.3007749319076538,
+      "learning_rate": 0.00040591567023285085,
+      "loss": 2.0135,
+      "mean_token_accuracy": 0.628263184428215,
+      "num_tokens": 902240.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "eval_loss": 2.0190296173095703,
+      "eval_mean_token_accuracy": 0.6285779621601105,
+      "eval_num_tokens": 902240.0,
+      "eval_runtime": 31.7727,
+      "eval_samples_per_second": 31.474,
+      "eval_steps_per_second": 7.868,
+      "step": 400
+    },
+    {
+      "epoch": 0.7288888888888889,
+      "grad_norm": 1.1896709203720093,
+      "learning_rate": 0.0004027690371302706,
+      "loss": 2.0396,
+      "mean_token_accuracy": 0.6259367674589157,
+      "num_tokens": 925091.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.3017927408218384,
+      "learning_rate": 0.0003996224040276904,
+      "loss": 2.0575,
+      "mean_token_accuracy": 0.6207560569047927,
+      "num_tokens": 948151.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.7644444444444445,
+      "grad_norm": 1.1772371530532837,
+      "learning_rate": 0.00039647577092511013,
+      "loss": 1.9884,
+      "mean_token_accuracy": 0.6323661148548126,
+      "num_tokens": 970339.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.7822222222222223,
+      "grad_norm": 1.0881459712982178,
+      "learning_rate": 0.0003933291378225299,
+      "loss": 2.0283,
+      "mean_token_accuracy": 0.6244203582406044,
+      "num_tokens": 993552.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1693931818008423,
+      "learning_rate": 0.00039018250471994963,
+      "loss": 2.0475,
+      "mean_token_accuracy": 0.6222739323973656,
+      "num_tokens": 1015564.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.8177777777777778,
+      "grad_norm": 1.2372338771820068,
+      "learning_rate": 0.0003870358716173694,
+      "loss": 1.9537,
+      "mean_token_accuracy": 0.6377866417169571,
+      "num_tokens": 1037721.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.8355555555555556,
+      "grad_norm": 1.1657993793487549,
+      "learning_rate": 0.0003838892385147892,
+      "loss": 2.0209,
+      "mean_token_accuracy": 0.6264078587293624,
+      "num_tokens": 1060627.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.1275551319122314,
+      "learning_rate": 0.00038074260541220897,
+      "loss": 1.9973,
+      "mean_token_accuracy": 0.6285641089081764,
+      "num_tokens": 1082672.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.8711111111111111,
+      "grad_norm": 1.162360429763794,
+      "learning_rate": 0.00037759597230962874,
+      "loss": 2.0117,
+      "mean_token_accuracy": 0.629520146548748,
+      "num_tokens": 1104857.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.1501044034957886,
+      "learning_rate": 0.00037444933920704847,
+      "loss": 1.9958,
+      "mean_token_accuracy": 0.6280175432562828,
+      "num_tokens": 1127612.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.1588389873504639,
+      "learning_rate": 0.0003713027061044682,
+      "loss": 2.0001,
+      "mean_token_accuracy": 0.6282740488648415,
+      "num_tokens": 1150411.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.9244444444444444,
+      "grad_norm": 1.1372511386871338,
+      "learning_rate": 0.00036815607300188797,
+      "loss": 2.0194,
+      "mean_token_accuracy": 0.6251062974333763,
+      "num_tokens": 1172683.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.9422222222222222,
+      "grad_norm": 1.1625277996063232,
+      "learning_rate": 0.00036500943989930775,
+      "loss": 1.9194,
+      "mean_token_accuracy": 0.6430759638547897,
+      "num_tokens": 1194160.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.2204004526138306,
+      "learning_rate": 0.00036186280679672753,
+      "loss": 1.9411,
+      "mean_token_accuracy": 0.6433150470256805,
+      "num_tokens": 1215760.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.9777777777777777,
+      "grad_norm": 1.1138572692871094,
+      "learning_rate": 0.00035871617369414725,
+      "loss": 2.0082,
+      "mean_token_accuracy": 0.6322961866855621,
+      "num_tokens": 1238191.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.9955555555555555,
+      "grad_norm": 1.1804752349853516,
+      "learning_rate": 0.00035556954059156703,
+      "loss": 1.9751,
+      "mean_token_accuracy": 0.6305457279086113,
+      "num_tokens": 1260429.0,
+      "step": 560
+    },
+    {
+      "epoch": 1.0124444444444445,
+      "grad_norm": 1.0810185670852661,
+      "learning_rate": 0.0003524229074889868,
+      "loss": 1.8437,
+      "mean_token_accuracy": 0.6523516507525193,
+      "num_tokens": 1281232.0,
+      "step": 570
+    },
+    {
+      "epoch": 1.0302222222222222,
+      "grad_norm": 1.2796133756637573,
+      "learning_rate": 0.0003492762743864066,
+      "loss": 1.7348,
+      "mean_token_accuracy": 0.6675047099590301,
+      "num_tokens": 1304130.0,
+      "step": 580
+    },
+    {
+      "epoch": 1.048,
+      "grad_norm": 1.1196218729019165,
+      "learning_rate": 0.0003461296412838263,
+      "loss": 1.7244,
+      "mean_token_accuracy": 0.6726136475801467,
+      "num_tokens": 1326753.0,
+      "step": 590
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "grad_norm": 1.1148934364318848,
+      "learning_rate": 0.00034298300818124604,
+      "loss": 1.7216,
+      "mean_token_accuracy": 0.6700241908431053,
+      "num_tokens": 1348985.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "eval_loss": 1.9544994831085205,
+      "eval_mean_token_accuracy": 0.638362931728363,
+      "eval_num_tokens": 1348985.0,
+      "eval_runtime": 30.1088,
+      "eval_samples_per_second": 33.213,
+      "eval_steps_per_second": 8.303,
+      "step": 600
+    },
+    {
+      "epoch": 1.0835555555555556,
+      "grad_norm": 1.0550259351730347,
+      "learning_rate": 0.0003398363750786658,
+      "loss": 1.734,
+      "mean_token_accuracy": 0.670471416413784,
+      "num_tokens": 1371318.0,
+      "step": 610
+    },
+    {
+      "epoch": 1.1013333333333333,
+      "grad_norm": 1.1968863010406494,
+      "learning_rate": 0.0003366897419760856,
+      "loss": 1.7819,
+      "mean_token_accuracy": 0.661465460062027,
+      "num_tokens": 1393816.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.1191111111111112,
+      "grad_norm": 1.2184885740280151,
+      "learning_rate": 0.0003335431088735054,
+      "loss": 1.769,
+      "mean_token_accuracy": 0.6628920555114746,
+      "num_tokens": 1416529.0,
+      "step": 630
+    },
+    {
+      "epoch": 1.1368888888888888,
+      "grad_norm": 1.2553062438964844,
+      "learning_rate": 0.00033039647577092515,
+      "loss": 1.7807,
+      "mean_token_accuracy": 0.6629297524690628,
+      "num_tokens": 1438970.0,
+      "step": 640
+    },
+    {
+      "epoch": 1.1546666666666667,
+      "grad_norm": 1.1857445240020752,
+      "learning_rate": 0.0003272498426683449,
+      "loss": 1.7555,
+      "mean_token_accuracy": 0.6634594619274139,
+      "num_tokens": 1461324.0,
+      "step": 650
+    },
+    {
+      "epoch": 1.1724444444444444,
+      "grad_norm": 1.2381750345230103,
+      "learning_rate": 0.0003241032095657646,
+      "loss": 1.7925,
+      "mean_token_accuracy": 0.6549596875905991,
+      "num_tokens": 1484246.0,
+      "step": 660
+    },
+    {
+      "epoch": 1.1902222222222223,
+      "grad_norm": 1.187692642211914,
+      "learning_rate": 0.0003209565764631844,
+      "loss": 1.7776,
+      "mean_token_accuracy": 0.658722423017025,
+      "num_tokens": 1507305.0,
+      "step": 670
+    },
+    {
+      "epoch": 1.208,
+      "grad_norm": 1.1748046875,
+      "learning_rate": 0.00031780994336060416,
+      "loss": 1.7355,
+      "mean_token_accuracy": 0.6706653699278832,
+      "num_tokens": 1529082.0,
+      "step": 680
+    },
+    {
+      "epoch": 1.2257777777777779,
+      "grad_norm": 1.2238432168960571,
+      "learning_rate": 0.00031466331025802394,
+      "loss": 1.7462,
+      "mean_token_accuracy": 0.6667303949594497,
+      "num_tokens": 1551159.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.2435555555555555,
+      "grad_norm": 1.1904925107955933,
+      "learning_rate": 0.00031151667715544366,
+      "loss": 1.7627,
+      "mean_token_accuracy": 0.6670840814709663,
+      "num_tokens": 1573348.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.2613333333333334,
+      "grad_norm": 1.282875895500183,
+      "learning_rate": 0.00030837004405286344,
+      "loss": 1.7625,
+      "mean_token_accuracy": 0.6660559684038162,
+      "num_tokens": 1595989.0,
+      "step": 710
+    },
+    {
+      "epoch": 1.279111111111111,
+      "grad_norm": 1.2079155445098877,
+      "learning_rate": 0.0003052234109502832,
+      "loss": 1.7979,
+      "mean_token_accuracy": 0.6636087343096733,
+      "num_tokens": 1617895.0,
+      "step": 720
+    },
+    {
+      "epoch": 1.2968888888888888,
+      "grad_norm": 1.1903313398361206,
+      "learning_rate": 0.000302076777847703,
+      "loss": 1.7552,
+      "mean_token_accuracy": 0.6641585499048233,
+      "num_tokens": 1640415.0,
+      "step": 730
+    },
+    {
+      "epoch": 1.3146666666666667,
+      "grad_norm": 1.1872097253799438,
+      "learning_rate": 0.0002989301447451227,
+      "loss": 1.74,
+      "mean_token_accuracy": 0.6656632766127586,
+      "num_tokens": 1662713.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.3324444444444445,
+      "grad_norm": 1.1994388103485107,
+      "learning_rate": 0.00029578351164254244,
+      "loss": 1.7757,
+      "mean_token_accuracy": 0.6637446776032447,
+      "num_tokens": 1684783.0,
+      "step": 750
+    },
+    {
+      "epoch": 1.3502222222222222,
+      "grad_norm": 1.2072243690490723,
+      "learning_rate": 0.0002926368785399622,
+      "loss": 1.7702,
+      "mean_token_accuracy": 0.6637947261333466,
+      "num_tokens": 1706623.0,
+      "step": 760
+    },
+    {
+      "epoch": 1.3679999999999999,
+      "grad_norm": 1.2028874158859253,
+      "learning_rate": 0.000289490245437382,
+      "loss": 1.7952,
+      "mean_token_accuracy": 0.6561207175254822,
+      "num_tokens": 1729494.0,
+      "step": 770
+    },
+    {
+      "epoch": 1.3857777777777778,
+      "grad_norm": 1.2276169061660767,
+      "learning_rate": 0.0002863436123348018,
+      "loss": 1.7624,
+      "mean_token_accuracy": 0.664546848833561,
+      "num_tokens": 1752045.0,
+      "step": 780
+    },
+    {
+      "epoch": 1.4035555555555557,
+      "grad_norm": 1.1520951986312866,
+      "learning_rate": 0.00028319697923222156,
+      "loss": 1.7743,
+      "mean_token_accuracy": 0.6641672790050507,
+      "num_tokens": 1775601.0,
+      "step": 790
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "grad_norm": 1.219901204109192,
+      "learning_rate": 0.0002800503461296413,
+      "loss": 1.7346,
+      "mean_token_accuracy": 0.6647952407598495,
+      "num_tokens": 1798633.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "eval_loss": 1.9113014936447144,
+      "eval_mean_token_accuracy": 0.6443024153709411,
+      "eval_num_tokens": 1798633.0,
+      "eval_runtime": 30.1829,
+      "eval_samples_per_second": 33.131,
+      "eval_steps_per_second": 8.283,
+      "step": 800
+    },
+    {
+      "epoch": 1.439111111111111,
+      "grad_norm": 1.2413419485092163,
+      "learning_rate": 0.00027690371302706106,
+      "loss": 1.7157,
+      "mean_token_accuracy": 0.6692047983407974,
+      "num_tokens": 1820656.0,
+      "step": 810
+    },
+    {
+      "epoch": 1.456888888888889,
+      "grad_norm": 1.2917033433914185,
+      "learning_rate": 0.0002737570799244808,
+      "loss": 1.7631,
+      "mean_token_accuracy": 0.6633412629365921,
+      "num_tokens": 1842871.0,
+      "step": 820
+    },
+    {
+      "epoch": 1.4746666666666668,
+      "grad_norm": 1.2017630338668823,
+      "learning_rate": 0.00027061044682190056,
+      "loss": 1.726,
+      "mean_token_accuracy": 0.6701689288020134,
+      "num_tokens": 1865652.0,
+      "step": 830
+    },
+    {
+      "epoch": 1.4924444444444445,
+      "grad_norm": 1.1332948207855225,
+      "learning_rate": 0.00026746381371932034,
+      "loss": 1.7561,
+      "mean_token_accuracy": 0.6648476481437683,
+      "num_tokens": 1888455.0,
+      "step": 840
+    },
+    {
+      "epoch": 1.5102222222222221,
+      "grad_norm": 1.2592897415161133,
+      "learning_rate": 0.00026431718061674007,
+      "loss": 1.7194,
+      "mean_token_accuracy": 0.6733420416712761,
+      "num_tokens": 1910676.0,
+      "step": 850
+    },
+    {
+      "epoch": 1.528,
+      "grad_norm": 1.2740918397903442,
+      "learning_rate": 0.00026117054751415985,
+      "loss": 1.7502,
+      "mean_token_accuracy": 0.6677413523197174,
+      "num_tokens": 1933253.0,
+      "step": 860
+    },
+    {
+      "epoch": 1.545777777777778,
+      "grad_norm": 1.2723543643951416,
+      "learning_rate": 0.0002580239144115796,
+      "loss": 1.7622,
+      "mean_token_accuracy": 0.6622136697173119,
+      "num_tokens": 1955820.0,
+      "step": 870
+    },
+    {
+      "epoch": 1.5635555555555556,
+      "grad_norm": 1.2220773696899414,
+      "learning_rate": 0.0002548772813089994,
+      "loss": 1.7568,
+      "mean_token_accuracy": 0.6640823170542717,
+      "num_tokens": 1978120.0,
+      "step": 880
+    },
+    {
+      "epoch": 1.5813333333333333,
+      "grad_norm": 1.2161861658096313,
+      "learning_rate": 0.0002517306482064191,
+      "loss": 1.7031,
+      "mean_token_accuracy": 0.6733875796198845,
+      "num_tokens": 2000508.0,
+      "step": 890
+    },
+    {
+      "epoch": 1.5991111111111111,
+      "grad_norm": 1.1472899913787842,
+      "learning_rate": 0.0002485840151038389,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6644304901361465,
+      "num_tokens": 2023356.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.616888888888889,
+      "grad_norm": 1.177524447441101,
+      "learning_rate": 0.00024543738200125863,
+      "loss": 1.6951,
+      "mean_token_accuracy": 0.6781735569238663,
+      "num_tokens": 2045965.0,
+      "step": 910
+    },
+    {
+      "epoch": 1.6346666666666667,
+      "grad_norm": 1.158720850944519,
+      "learning_rate": 0.0002422907488986784,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6616655588150024,
+      "num_tokens": 2069108.0,
+      "step": 920
+    },
+    {
+      "epoch": 1.6524444444444444,
+      "grad_norm": 1.3048863410949707,
+      "learning_rate": 0.0002391441157960982,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6625866696238518,
+      "num_tokens": 2091755.0,
+      "step": 930
+    },
+    {
+      "epoch": 1.6702222222222223,
+      "grad_norm": 1.3040592670440674,
+      "learning_rate": 0.00023599748269351794,
+      "loss": 1.7286,
+      "mean_token_accuracy": 0.6688349068164825,
+      "num_tokens": 2114496.0,
+      "step": 940
+    },
+    {
+      "epoch": 1.688,
+      "grad_norm": 1.1945664882659912,
+      "learning_rate": 0.0002328508495909377,
+      "loss": 1.7479,
+      "mean_token_accuracy": 0.6634002208709717,
+      "num_tokens": 2137041.0,
+      "step": 950
+    },
+    {
+      "epoch": 1.7057777777777776,
+      "grad_norm": 1.2552154064178467,
+      "learning_rate": 0.00022970421648835747,
+      "loss": 1.7537,
+      "mean_token_accuracy": 0.6634084343910217,
+      "num_tokens": 2159711.0,
+      "step": 960
+    },
+    {
+      "epoch": 1.7235555555555555,
+      "grad_norm": 1.2484403848648071,
+      "learning_rate": 0.00022655758338577722,
+      "loss": 1.7557,
+      "mean_token_accuracy": 0.6687701687216758,
+      "num_tokens": 2181675.0,
+      "step": 970
+    },
+    {
+      "epoch": 1.7413333333333334,
+      "grad_norm": 1.1729741096496582,
+      "learning_rate": 0.000223410950283197,
+      "loss": 1.7274,
+      "mean_token_accuracy": 0.6730923235416413,
+      "num_tokens": 2204671.0,
+      "step": 980
+    },
+    {
+      "epoch": 1.759111111111111,
+      "grad_norm": 1.312406301498413,
+      "learning_rate": 0.00022026431718061672,
+      "loss": 1.7377,
+      "mean_token_accuracy": 0.6612286046147346,
+      "num_tokens": 2227866.0,
+      "step": 990
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "grad_norm": 1.3160700798034668,
+      "learning_rate": 0.0002171176840780365,
+      "loss": 1.7224,
+      "mean_token_accuracy": 0.6710903704166412,
+      "num_tokens": 2250379.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "eval_loss": 1.8698750734329224,
+      "eval_mean_token_accuracy": 0.6504300956726075,
+      "eval_num_tokens": 2250379.0,
+      "eval_runtime": 30.1651,
+      "eval_samples_per_second": 33.151,
+      "eval_steps_per_second": 8.288,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7946666666666666,
+      "grad_norm": 1.1682268381118774,
+      "learning_rate": 0.00021397105097545628,
+      "loss": 1.7458,
+      "mean_token_accuracy": 0.665731918811798,
+      "num_tokens": 2272999.0,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8124444444444445,
+      "grad_norm": 1.1300792694091797,
+      "learning_rate": 0.00021082441787287603,
+      "loss": 1.7261,
+      "mean_token_accuracy": 0.6688766568899155,
+      "num_tokens": 2295110.0,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8302222222222222,
+      "grad_norm": 1.2450717687606812,
+      "learning_rate": 0.00020767778477029578,
+      "loss": 1.6968,
+      "mean_token_accuracy": 0.6718593999743462,
+      "num_tokens": 2317432.0,
+      "step": 1030
+    },
+    {
+      "epoch": 1.8479999999999999,
+      "grad_norm": 1.201785683631897,
+      "learning_rate": 0.00020453115166771556,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6636266440153122,
+      "num_tokens": 2340305.0,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8657777777777778,
+      "grad_norm": 1.2161189317703247,
+      "learning_rate": 0.0002013845185651353,
+      "loss": 1.7335,
+      "mean_token_accuracy": 0.6697820112109184,
+      "num_tokens": 2362746.0,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8835555555555556,
+      "grad_norm": 1.209251046180725,
+      "learning_rate": 0.00019823788546255506,
+      "loss": 1.7197,
+      "mean_token_accuracy": 0.6663407534360886,
+      "num_tokens": 2385647.0,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9013333333333333,
+      "grad_norm": 1.1715242862701416,
+      "learning_rate": 0.00019509125235997482,
+      "loss": 1.7298,
+      "mean_token_accuracy": 0.6650380924344063,
+      "num_tokens": 2408732.0,
+      "step": 1070
+    },
+    {
+      "epoch": 1.919111111111111,
+      "grad_norm": 1.2717958688735962,
+      "learning_rate": 0.0001919446192573946,
+      "loss": 1.7101,
+      "mean_token_accuracy": 0.672779667377472,
+      "num_tokens": 2431137.0,
+      "step": 1080
+    },
+    {
+      "epoch": 1.9368888888888889,
+      "grad_norm": 1.2264835834503174,
+      "learning_rate": 0.00018879798615481437,
+      "loss": 1.6754,
+      "mean_token_accuracy": 0.6768045455217362,
+      "num_tokens": 2453517.0,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9546666666666668,
+      "grad_norm": 1.3289543390274048,
+      "learning_rate": 0.0001856513530522341,
+      "loss": 1.7178,
+      "mean_token_accuracy": 0.6685071781277656,
+      "num_tokens": 2475737.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9724444444444444,
+      "grad_norm": 1.1367521286010742,
+      "learning_rate": 0.00018250471994965388,
+      "loss": 1.7109,
+      "mean_token_accuracy": 0.6677573248744011,
+      "num_tokens": 2497880.0,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9902222222222221,
+      "grad_norm": 1.246870994567871,
+      "learning_rate": 0.00017935808684707363,
+      "loss": 1.7317,
+      "mean_token_accuracy": 0.6695448830723763,
+      "num_tokens": 2519974.0,
+      "step": 1120
+    },
+    {
+      "epoch": 2.007111111111111,
+      "grad_norm": 1.0757529735565186,
+      "learning_rate": 0.0001762114537444934,
+      "loss": 1.586,
+      "mean_token_accuracy": 0.6983135671992051,
+      "num_tokens": 2540296.0,
+      "step": 1130
+    },
+    {
+      "epoch": 2.024888888888889,
+      "grad_norm": 1.3501553535461426,
+      "learning_rate": 0.00017306482064191316,
+      "loss": 1.4761,
+      "mean_token_accuracy": 0.7080946862697601,
+      "num_tokens": 2561919.0,
+      "step": 1140
+    },
+    {
+      "epoch": 2.042666666666667,
+      "grad_norm": 1.502793788909912,
+      "learning_rate": 0.0001699181875393329,
+      "loss": 1.5049,
+      "mean_token_accuracy": 0.7034129843115806,
+      "num_tokens": 2584915.0,
+      "step": 1150
+    },
+    {
+      "epoch": 2.0604444444444443,
+      "grad_norm": 1.2668856382369995,
+      "learning_rate": 0.0001667715544367527,
+      "loss": 1.4612,
+      "mean_token_accuracy": 0.7100765943527222,
+      "num_tokens": 2606857.0,
+      "step": 1160
+    },
+    {
+      "epoch": 2.078222222222222,
+      "grad_norm": 1.386775016784668,
+      "learning_rate": 0.00016362492133417244,
+      "loss": 1.4695,
+      "mean_token_accuracy": 0.7058908969163895,
+      "num_tokens": 2629737.0,
+      "step": 1170
+    },
+    {
+      "epoch": 2.096,
+      "grad_norm": 1.3320142030715942,
+      "learning_rate": 0.0001604782882315922,
+      "loss": 1.4337,
+      "mean_token_accuracy": 0.7136739388108253,
+      "num_tokens": 2651936.0,
+      "step": 1180
+    },
+    {
+      "epoch": 2.113777777777778,
+      "grad_norm": 1.289437174797058,
+      "learning_rate": 0.00015733165512901197,
+      "loss": 1.4975,
+      "mean_token_accuracy": 0.7037445783615113,
+      "num_tokens": 2675241.0,
+      "step": 1190
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "grad_norm": 1.208194613456726,
+      "learning_rate": 0.00015418502202643172,
+      "loss": 1.4769,
+      "mean_token_accuracy": 0.7093130722641945,
+      "num_tokens": 2698219.0,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "eval_loss": 1.8875290155410767,
+      "eval_mean_token_accuracy": 0.6501186113357544,
+      "eval_num_tokens": 2698219.0,
+      "eval_runtime": 30.175,
+      "eval_samples_per_second": 33.14,
+      "eval_steps_per_second": 8.285,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1493333333333333,
+      "grad_norm": 1.360378623008728,
+      "learning_rate": 0.0001510383889238515,
+      "loss": 1.485,
+      "mean_token_accuracy": 0.7059277340769767,
+      "num_tokens": 2721580.0,
+      "step": 1210
+    },
+    {
+      "epoch": 2.167111111111111,
+      "grad_norm": 1.3289263248443604,
+      "learning_rate": 0.00014789175582127122,
+      "loss": 1.4869,
+      "mean_token_accuracy": 0.7061908945441246,
+      "num_tokens": 2744110.0,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1848888888888887,
+      "grad_norm": 1.2693774700164795,
+      "learning_rate": 0.000144745122718691,
+      "loss": 1.4555,
+      "mean_token_accuracy": 0.712950111925602,
+      "num_tokens": 2765844.0,
+      "step": 1230
+    },
+    {
+      "epoch": 2.2026666666666666,
+      "grad_norm": 1.3220160007476807,
+      "learning_rate": 0.00014159848961611078,
+      "loss": 1.4634,
+      "mean_token_accuracy": 0.7090483531355858,
+      "num_tokens": 2788070.0,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2204444444444444,
+      "grad_norm": 1.2682182788848877,
+      "learning_rate": 0.00013845185651353053,
+      "loss": 1.497,
+      "mean_token_accuracy": 0.7039189741015435,
+      "num_tokens": 2810964.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.2382222222222223,
+      "grad_norm": 1.3531242609024048,
+      "learning_rate": 0.00013530522341095028,
+      "loss": 1.4625,
+      "mean_token_accuracy": 0.7105821236968041,
+      "num_tokens": 2833176.0,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.2819761037826538,
+      "learning_rate": 0.00013215859030837003,
+      "loss": 1.5092,
+      "mean_token_accuracy": 0.7021819025278091,
+      "num_tokens": 2855440.0,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2737777777777777,
+      "grad_norm": 1.3131153583526611,
+      "learning_rate": 0.0001290119572057898,
+      "loss": 1.4458,
+      "mean_token_accuracy": 0.7121976673603058,
+      "num_tokens": 2877713.0,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2915555555555556,
+      "grad_norm": 1.2332444190979004,
+      "learning_rate": 0.00012586532410320956,
+      "loss": 1.4797,
+      "mean_token_accuracy": 0.7103414624929428,
+      "num_tokens": 2899934.0,
+      "step": 1290
+    },
+    {
+      "epoch": 2.3093333333333335,
+      "grad_norm": 1.2947789430618286,
+      "learning_rate": 0.00012271869100062931,
+      "loss": 1.4969,
+      "mean_token_accuracy": 0.7032593175768852,
+      "num_tokens": 2923244.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.327111111111111,
+      "grad_norm": 1.4002538919448853,
+      "learning_rate": 0.0001195720578980491,
+      "loss": 1.4538,
+      "mean_token_accuracy": 0.711962579190731,
+      "num_tokens": 2945580.0,
+      "step": 1310
+    },
+    {
+      "epoch": 2.344888888888889,
+      "grad_norm": 1.270635962486267,
+      "learning_rate": 0.00011642542479546884,
+      "loss": 1.4622,
+      "mean_token_accuracy": 0.7059817537665367,
+      "num_tokens": 2969470.0,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3626666666666667,
+      "grad_norm": 1.3131049871444702,
+      "learning_rate": 0.00011327879169288861,
+      "loss": 1.4597,
+      "mean_token_accuracy": 0.7036674708127976,
+      "num_tokens": 2991982.0,
+      "step": 1330
+    },
+    {
+      "epoch": 2.3804444444444446,
+      "grad_norm": 1.3708815574645996,
+      "learning_rate": 0.00011013215859030836,
+      "loss": 1.4837,
+      "mean_token_accuracy": 0.706661482155323,
+      "num_tokens": 3014206.0,
+      "step": 1340
+    },
+    {
+      "epoch": 2.398222222222222,
+      "grad_norm": 1.3198243379592896,
+      "learning_rate": 0.00010698552548772814,
+      "loss": 1.4431,
+      "mean_token_accuracy": 0.7101009368896485,
+      "num_tokens": 3036651.0,
+      "step": 1350
+    },
+    {
+      "epoch": 2.416,
+      "grad_norm": 1.280879259109497,
+      "learning_rate": 0.00010383889238514789,
+      "loss": 1.477,
+      "mean_token_accuracy": 0.7085638150572777,
+      "num_tokens": 3058745.0,
+      "step": 1360
+    },
+    {
+      "epoch": 2.433777777777778,
+      "grad_norm": 1.419290542602539,
+      "learning_rate": 0.00010069225928256766,
+      "loss": 1.4887,
+      "mean_token_accuracy": 0.7048677980899811,
+      "num_tokens": 3081726.0,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4515555555555557,
+      "grad_norm": 1.3684442043304443,
+      "learning_rate": 9.754562617998741e-05,
+      "loss": 1.4677,
+      "mean_token_accuracy": 0.7087965488433838,
+      "num_tokens": 3104292.0,
+      "step": 1380
+    },
+    {
+      "epoch": 2.469333333333333,
+      "grad_norm": 1.3453881740570068,
+      "learning_rate": 9.439899307740719e-05,
+      "loss": 1.4682,
+      "mean_token_accuracy": 0.7076600238680839,
+      "num_tokens": 3126334.0,
+      "step": 1390
+    },
+    {
+      "epoch": 2.487111111111111,
+      "grad_norm": 1.2759828567504883,
+      "learning_rate": 9.125235997482694e-05,
+      "loss": 1.4971,
+      "mean_token_accuracy": 0.7039492294192314,
+      "num_tokens": 3149845.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.487111111111111,
+      "eval_loss": 1.873150110244751,
+      "eval_mean_token_accuracy": 0.65266099858284,
+      "eval_num_tokens": 3149845.0,
+      "eval_runtime": 30.0464,
+      "eval_samples_per_second": 33.282,
+      "eval_steps_per_second": 8.32,
+      "step": 1400
+    },
+    {
+      "epoch": 2.504888888888889,
+      "grad_norm": 1.3677406311035156,
+      "learning_rate": 8.81057268722467e-05,
+      "loss": 1.4905,
+      "mean_token_accuracy": 0.7065914765000343,
+      "num_tokens": 3172117.0,
+      "step": 1410
+    },
+    {
+      "epoch": 2.522666666666667,
+      "grad_norm": 1.2730094194412231,
+      "learning_rate": 8.495909376966645e-05,
+      "loss": 1.4772,
+      "mean_token_accuracy": 0.7116217255592346,
+      "num_tokens": 3194261.0,
+      "step": 1420
+    },
+    {
+      "epoch": 2.5404444444444443,
+      "grad_norm": 1.3448493480682373,
+      "learning_rate": 8.181246066708622e-05,
+      "loss": 1.3895,
+      "mean_token_accuracy": 0.7193023800849915,
+      "num_tokens": 3216455.0,
+      "step": 1430
+    },
+    {
+      "epoch": 2.558222222222222,
+      "grad_norm": 1.3793234825134277,
+      "learning_rate": 7.866582756450598e-05,
+      "loss": 1.4958,
+      "mean_token_accuracy": 0.7059502750635147,
+      "num_tokens": 3238359.0,
+      "step": 1440
+    },
+    {
+      "epoch": 2.576,
+      "grad_norm": 1.3225526809692383,
+      "learning_rate": 7.551919446192575e-05,
+      "loss": 1.5047,
+      "mean_token_accuracy": 0.7010831996798516,
+      "num_tokens": 3261568.0,
+      "step": 1450
+    },
+    {
+      "epoch": 2.5937777777777775,
+      "grad_norm": 1.326993465423584,
+      "learning_rate": 7.23725613593455e-05,
+      "loss": 1.5066,
+      "mean_token_accuracy": 0.6991274550557136,
+      "num_tokens": 3284517.0,
+      "step": 1460
+    },
+    {
+      "epoch": 2.6115555555555554,
+      "grad_norm": 1.3944668769836426,
+      "learning_rate": 6.922592825676527e-05,
+      "loss": 1.461,
+      "mean_token_accuracy": 0.7067686557769776,
+      "num_tokens": 3306623.0,
+      "step": 1470
+    },
+    {
+      "epoch": 2.6293333333333333,
+      "grad_norm": 1.2688894271850586,
+      "learning_rate": 6.607929515418502e-05,
+      "loss": 1.4464,
+      "mean_token_accuracy": 0.7143596678972244,
+      "num_tokens": 3329270.0,
+      "step": 1480
+    },
+    {
+      "epoch": 2.647111111111111,
+      "grad_norm": 1.245979905128479,
+      "learning_rate": 6.293266205160478e-05,
+      "loss": 1.4745,
+      "mean_token_accuracy": 0.7083328410983085,
+      "num_tokens": 3352533.0,
+      "step": 1490
+    },
+    {
+      "epoch": 2.664888888888889,
+      "grad_norm": 1.359151840209961,
+      "learning_rate": 5.978602894902455e-05,
+      "loss": 1.4547,
+      "mean_token_accuracy": 0.7116006925702095,
+      "num_tokens": 3374132.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.6826666666666665,
+      "grad_norm": 1.3699895143508911,
+      "learning_rate": 5.6639395846444305e-05,
+      "loss": 1.4876,
+      "mean_token_accuracy": 0.702751100063324,
+      "num_tokens": 3397176.0,
+      "step": 1510
+    },
+    {
+      "epoch": 2.7004444444444444,
+      "grad_norm": 1.3784525394439697,
+      "learning_rate": 5.349276274386407e-05,
+      "loss": 1.4384,
+      "mean_token_accuracy": 0.713467463850975,
+      "num_tokens": 3418961.0,
+      "step": 1520
+    },
+    {
+      "epoch": 2.7182222222222223,
+      "grad_norm": 1.3480818271636963,
+      "learning_rate": 5.034612964128383e-05,
+      "loss": 1.4436,
+      "mean_token_accuracy": 0.7122948378324508,
+      "num_tokens": 3441815.0,
+      "step": 1530
+    },
+    {
+      "epoch": 2.7359999999999998,
+      "grad_norm": 1.2246533632278442,
+      "learning_rate": 4.719949653870359e-05,
+      "loss": 1.4695,
+      "mean_token_accuracy": 0.70788504332304,
+      "num_tokens": 3464582.0,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7537777777777777,
+      "grad_norm": 1.2388625144958496,
+      "learning_rate": 4.405286343612335e-05,
+      "loss": 1.433,
+      "mean_token_accuracy": 0.7149359166622162,
+      "num_tokens": 3486694.0,
+      "step": 1550
+    },
+    {
+      "epoch": 2.7715555555555556,
+      "grad_norm": 1.317224383354187,
+      "learning_rate": 4.090623033354311e-05,
+      "loss": 1.4772,
+      "mean_token_accuracy": 0.7114273443818092,
+      "num_tokens": 3508908.0,
+      "step": 1560
+    },
+    {
+      "epoch": 2.7893333333333334,
+      "grad_norm": 1.2926599979400635,
+      "learning_rate": 3.7759597230962874e-05,
+      "loss": 1.425,
+      "mean_token_accuracy": 0.7173908859491348,
+      "num_tokens": 3530306.0,
+      "step": 1570
+    },
+    {
+      "epoch": 2.8071111111111113,
+      "grad_norm": 1.3763422966003418,
+      "learning_rate": 3.461296412838263e-05,
+      "loss": 1.4996,
+      "mean_token_accuracy": 0.7037103369832038,
+      "num_tokens": 3552880.0,
+      "step": 1580
+    },
+    {
+      "epoch": 2.824888888888889,
+      "grad_norm": 1.3559826612472534,
+      "learning_rate": 3.146633102580239e-05,
+      "loss": 1.4756,
+      "mean_token_accuracy": 0.7092328786849975,
+      "num_tokens": 3575800.0,
+      "step": 1590
+    },
+    {
+      "epoch": 2.8426666666666667,
+      "grad_norm": 1.3408883810043335,
+      "learning_rate": 2.8319697923222152e-05,
+      "loss": 1.4735,
+      "mean_token_accuracy": 0.7120713770389557,
+      "num_tokens": 3598214.0,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8426666666666667,
+      "eval_loss": 1.8612734079360962,
+      "eval_mean_token_accuracy": 0.6546365599632263,
+      "eval_num_tokens": 3598214.0,
+      "eval_runtime": 30.1859,
+      "eval_samples_per_second": 33.128,
+      "eval_steps_per_second": 8.282,
+      "step": 1600
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1689,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2669174249603072e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1689/README.md

@@ -0,0 +1,202 @@
+---
+base_model: naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

checkpoint-1689/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-1689/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

checkpoint-1689/chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-1689/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc0c7a7288e190a96caa8201f7eaf5a6809b17b8a040ec01265293e35ba25154
+size 78925378

checkpoint-1689/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:622064fb68e7107ee7b349ab074bd33170144438520452d68e43e21e6486d978
+size 14244

checkpoint-1689/special_tokens_map.json

@@ -0,0 +1,86 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endofturn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1689/tokenizer_config.json

@@ -0,0 +1,501 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|_unuse_missing_100256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|_unuse_missing_100261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|_unuse_missing_100262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|_unuse_missing_100263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|_unuse_missing_100264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|_unuse_missing_100265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|_unuse_missing_100266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|_unuse_missing_100267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|_unuse_missing_100268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|_unuse_missing_100269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|_unuse_missing_100270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|_unuse_missing_100271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|stop|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|endofturn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110491": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110492": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110493": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110494": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110495": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110496": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110497": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110498": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110499": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110500": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110501": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110502": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110503": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110504": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110505": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110506": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110507": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110508": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110509": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110510": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110511": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110512": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110513": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110514": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110515": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110516": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110517": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110518": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110519": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110520": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110521": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110522": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110523": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endofturn|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-1689/trainer_state.json

@@ -0,0 +1,1626 @@
+{
+  "best_global_step": 1600,
+  "best_metric": 1.8612734079360962,
+  "best_model_checkpoint": "/content/drive/MyDrive/hyperclova-deobfuscation-lora/checkpoint-1600",
+  "epoch": 3.0,
+  "eval_steps": 200,
+  "global_step": 1689,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017777777777777778,
+      "grad_norm": 3.0164783000946045,
+      "learning_rate": 4.4999999999999996e-05,
+      "loss": 4.0476,
+      "mean_token_accuracy": 0.3606465607881546,
+      "num_tokens": 22106.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.035555555555555556,
+      "grad_norm": 2.489065408706665,
+      "learning_rate": 9.5e-05,
+      "loss": 3.3625,
+      "mean_token_accuracy": 0.46429387107491493,
+      "num_tokens": 44943.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 1.6735386848449707,
+      "learning_rate": 0.000145,
+      "loss": 2.9752,
+      "mean_token_accuracy": 0.5158864699304104,
+      "num_tokens": 67397.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.07111111111111111,
+      "grad_norm": 1.5050151348114014,
+      "learning_rate": 0.00019500000000000002,
+      "loss": 2.7549,
+      "mean_token_accuracy": 0.5370393514633178,
+      "num_tokens": 89803.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.3750827312469482,
+      "learning_rate": 0.000245,
+      "loss": 2.6264,
+      "mean_token_accuracy": 0.550141765922308,
+      "num_tokens": 112364.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.10666666666666667,
+      "grad_norm": 1.4109346866607666,
+      "learning_rate": 0.000295,
+      "loss": 2.5264,
+      "mean_token_accuracy": 0.5618108093738556,
+      "num_tokens": 134028.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.12444444444444444,
+      "grad_norm": 1.3981140851974487,
+      "learning_rate": 0.000345,
+      "loss": 2.4839,
+      "mean_token_accuracy": 0.5678502731025219,
+      "num_tokens": 156703.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.14222222222222222,
+      "grad_norm": 1.4779222011566162,
+      "learning_rate": 0.000395,
+      "loss": 2.4262,
+      "mean_token_accuracy": 0.5784575901925564,
+      "num_tokens": 178986.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.1966180801391602,
+      "learning_rate": 0.00044500000000000003,
+      "loss": 2.4566,
+      "mean_token_accuracy": 0.5695327177643776,
+      "num_tokens": 201989.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.4000093936920166,
+      "learning_rate": 0.000495,
+      "loss": 2.3821,
+      "mean_token_accuracy": 0.5801047712564469,
+      "num_tokens": 223936.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.19555555555555557,
+      "grad_norm": 1.3838802576065063,
+      "learning_rate": 0.0004971680302076778,
+      "loss": 2.3444,
+      "mean_token_accuracy": 0.5860289096832275,
+      "num_tokens": 246101.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.3671435117721558,
+      "learning_rate": 0.0004940213971050976,
+      "loss": 2.3484,
+      "mean_token_accuracy": 0.5840608969330787,
+      "num_tokens": 269187.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.2311111111111111,
+      "grad_norm": 1.0929336547851562,
+      "learning_rate": 0.0004908747640025173,
+      "loss": 2.2781,
+      "mean_token_accuracy": 0.5930660709738731,
+      "num_tokens": 291454.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.24888888888888888,
+      "grad_norm": 1.1275103092193604,
+      "learning_rate": 0.00048772813089993704,
+      "loss": 2.2775,
+      "mean_token_accuracy": 0.5900422058999538,
+      "num_tokens": 314204.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.1598752737045288,
+      "learning_rate": 0.0004845814977973568,
+      "loss": 2.242,
+      "mean_token_accuracy": 0.5971075922250748,
+      "num_tokens": 337167.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.28444444444444444,
+      "grad_norm": 1.2944793701171875,
+      "learning_rate": 0.0004814348646947766,
+      "loss": 2.2755,
+      "mean_token_accuracy": 0.5881656795740128,
+      "num_tokens": 360272.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.3022222222222222,
+      "grad_norm": 1.2451213598251343,
+      "learning_rate": 0.0004782882315921964,
+      "loss": 2.1925,
+      "mean_token_accuracy": 0.6079613700509071,
+      "num_tokens": 382779.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.168745517730713,
+      "learning_rate": 0.00047514159848961615,
+      "loss": 2.1997,
+      "mean_token_accuracy": 0.6048969209194184,
+      "num_tokens": 405438.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.3377777777777778,
+      "grad_norm": 1.2053375244140625,
+      "learning_rate": 0.0004719949653870359,
+      "loss": 2.236,
+      "mean_token_accuracy": 0.5962931856513023,
+      "num_tokens": 428660.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.1758276224136353,
+      "learning_rate": 0.00046884833228445566,
+      "loss": 2.1761,
+      "mean_token_accuracy": 0.6056372985243798,
+      "num_tokens": 450808.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "eval_loss": 2.166856527328491,
+      "eval_mean_token_accuracy": 0.6084163203239441,
+      "eval_num_tokens": 450808.0,
+      "eval_runtime": 30.2794,
+      "eval_samples_per_second": 33.026,
+      "eval_steps_per_second": 8.256,
+      "step": 200
+    },
+    {
+      "epoch": 0.37333333333333335,
+      "grad_norm": 1.3344407081604004,
+      "learning_rate": 0.0004657016991818754,
+      "loss": 2.1878,
+      "mean_token_accuracy": 0.6024597108364105,
+      "num_tokens": 473434.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.39111111111111113,
+      "grad_norm": 1.1835261583328247,
+      "learning_rate": 0.00046255506607929516,
+      "loss": 2.2188,
+      "mean_token_accuracy": 0.5974872022867203,
+      "num_tokens": 496482.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.4088888888888889,
+      "grad_norm": 1.1516607999801636,
+      "learning_rate": 0.00045940843297671494,
+      "loss": 2.1659,
+      "mean_token_accuracy": 0.604978509247303,
+      "num_tokens": 519379.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.1609504222869873,
+      "learning_rate": 0.00045626179987413466,
+      "loss": 2.1777,
+      "mean_token_accuracy": 0.6044537022709846,
+      "num_tokens": 542631.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.1110681295394897,
+      "learning_rate": 0.00045311516677155444,
+      "loss": 2.1604,
+      "mean_token_accuracy": 0.6043903902173042,
+      "num_tokens": 565400.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.4622222222222222,
+      "grad_norm": 1.2393710613250732,
+      "learning_rate": 0.0004499685336689742,
+      "loss": 2.0864,
+      "mean_token_accuracy": 0.6146551981568337,
+      "num_tokens": 588003.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.3670927286148071,
+      "learning_rate": 0.000446821900566394,
+      "loss": 2.1201,
+      "mean_token_accuracy": 0.6108790010213851,
+      "num_tokens": 610974.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.49777777777777776,
+      "grad_norm": 1.1501164436340332,
+      "learning_rate": 0.0004436752674638137,
+      "loss": 2.1437,
+      "mean_token_accuracy": 0.6097761034965515,
+      "num_tokens": 634093.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.5155555555555555,
+      "grad_norm": 1.1109156608581543,
+      "learning_rate": 0.00044052863436123345,
+      "loss": 2.0543,
+      "mean_token_accuracy": 0.6271390929818154,
+      "num_tokens": 656188.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.2343859672546387,
+      "learning_rate": 0.0004373820012586532,
+      "loss": 2.0971,
+      "mean_token_accuracy": 0.6169493526220322,
+      "num_tokens": 678769.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.5511111111111111,
+      "grad_norm": 1.083136796951294,
+      "learning_rate": 0.000434235368156073,
+      "loss": 2.0734,
+      "mean_token_accuracy": 0.6225788787007331,
+      "num_tokens": 701734.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.5688888888888889,
+      "grad_norm": 1.1497492790222168,
+      "learning_rate": 0.0004310887350534928,
+      "loss": 2.0603,
+      "mean_token_accuracy": 0.6220441117882729,
+      "num_tokens": 724815.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.1948379278182983,
+      "learning_rate": 0.00042794210195091256,
+      "loss": 2.0612,
+      "mean_token_accuracy": 0.6211912930011749,
+      "num_tokens": 746903.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.6044444444444445,
+      "grad_norm": 1.1575194597244263,
+      "learning_rate": 0.00042479546884833234,
+      "loss": 2.042,
+      "mean_token_accuracy": 0.6249620258808136,
+      "num_tokens": 768982.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.6222222222222222,
+      "grad_norm": 1.0737602710723877,
+      "learning_rate": 0.00042164883574575206,
+      "loss": 2.066,
+      "mean_token_accuracy": 0.6236828655004502,
+      "num_tokens": 791061.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.0655274391174316,
+      "learning_rate": 0.0004185022026431718,
+      "loss": 2.0556,
+      "mean_token_accuracy": 0.6238334342837334,
+      "num_tokens": 813112.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.6577777777777778,
+      "grad_norm": 1.2494139671325684,
+      "learning_rate": 0.00041535556954059156,
+      "loss": 2.0544,
+      "mean_token_accuracy": 0.623551732301712,
+      "num_tokens": 835469.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.6755555555555556,
+      "grad_norm": 1.251245379447937,
+      "learning_rate": 0.00041220893643801134,
+      "loss": 2.0623,
+      "mean_token_accuracy": 0.6137503892183304,
+      "num_tokens": 857795.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.6933333333333334,
+      "grad_norm": 1.1011890172958374,
+      "learning_rate": 0.0004090623033354311,
+      "loss": 2.0281,
+      "mean_token_accuracy": 0.6267385333776474,
+      "num_tokens": 879659.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 1.3007749319076538,
+      "learning_rate": 0.00040591567023285085,
+      "loss": 2.0135,
+      "mean_token_accuracy": 0.628263184428215,
+      "num_tokens": 902240.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "eval_loss": 2.0190296173095703,
+      "eval_mean_token_accuracy": 0.6285779621601105,
+      "eval_num_tokens": 902240.0,
+      "eval_runtime": 31.7727,
+      "eval_samples_per_second": 31.474,
+      "eval_steps_per_second": 7.868,
+      "step": 400
+    },
+    {
+      "epoch": 0.7288888888888889,
+      "grad_norm": 1.1896709203720093,
+      "learning_rate": 0.0004027690371302706,
+      "loss": 2.0396,
+      "mean_token_accuracy": 0.6259367674589157,
+      "num_tokens": 925091.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.3017927408218384,
+      "learning_rate": 0.0003996224040276904,
+      "loss": 2.0575,
+      "mean_token_accuracy": 0.6207560569047927,
+      "num_tokens": 948151.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.7644444444444445,
+      "grad_norm": 1.1772371530532837,
+      "learning_rate": 0.00039647577092511013,
+      "loss": 1.9884,
+      "mean_token_accuracy": 0.6323661148548126,
+      "num_tokens": 970339.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.7822222222222223,
+      "grad_norm": 1.0881459712982178,
+      "learning_rate": 0.0003933291378225299,
+      "loss": 2.0283,
+      "mean_token_accuracy": 0.6244203582406044,
+      "num_tokens": 993552.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.1693931818008423,
+      "learning_rate": 0.00039018250471994963,
+      "loss": 2.0475,
+      "mean_token_accuracy": 0.6222739323973656,
+      "num_tokens": 1015564.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.8177777777777778,
+      "grad_norm": 1.2372338771820068,
+      "learning_rate": 0.0003870358716173694,
+      "loss": 1.9537,
+      "mean_token_accuracy": 0.6377866417169571,
+      "num_tokens": 1037721.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.8355555555555556,
+      "grad_norm": 1.1657993793487549,
+      "learning_rate": 0.0003838892385147892,
+      "loss": 2.0209,
+      "mean_token_accuracy": 0.6264078587293624,
+      "num_tokens": 1060627.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.1275551319122314,
+      "learning_rate": 0.00038074260541220897,
+      "loss": 1.9973,
+      "mean_token_accuracy": 0.6285641089081764,
+      "num_tokens": 1082672.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.8711111111111111,
+      "grad_norm": 1.162360429763794,
+      "learning_rate": 0.00037759597230962874,
+      "loss": 2.0117,
+      "mean_token_accuracy": 0.629520146548748,
+      "num_tokens": 1104857.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.1501044034957886,
+      "learning_rate": 0.00037444933920704847,
+      "loss": 1.9958,
+      "mean_token_accuracy": 0.6280175432562828,
+      "num_tokens": 1127612.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.1588389873504639,
+      "learning_rate": 0.0003713027061044682,
+      "loss": 2.0001,
+      "mean_token_accuracy": 0.6282740488648415,
+      "num_tokens": 1150411.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.9244444444444444,
+      "grad_norm": 1.1372511386871338,
+      "learning_rate": 0.00036815607300188797,
+      "loss": 2.0194,
+      "mean_token_accuracy": 0.6251062974333763,
+      "num_tokens": 1172683.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.9422222222222222,
+      "grad_norm": 1.1625277996063232,
+      "learning_rate": 0.00036500943989930775,
+      "loss": 1.9194,
+      "mean_token_accuracy": 0.6430759638547897,
+      "num_tokens": 1194160.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.2204004526138306,
+      "learning_rate": 0.00036186280679672753,
+      "loss": 1.9411,
+      "mean_token_accuracy": 0.6433150470256805,
+      "num_tokens": 1215760.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.9777777777777777,
+      "grad_norm": 1.1138572692871094,
+      "learning_rate": 0.00035871617369414725,
+      "loss": 2.0082,
+      "mean_token_accuracy": 0.6322961866855621,
+      "num_tokens": 1238191.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.9955555555555555,
+      "grad_norm": 1.1804752349853516,
+      "learning_rate": 0.00035556954059156703,
+      "loss": 1.9751,
+      "mean_token_accuracy": 0.6305457279086113,
+      "num_tokens": 1260429.0,
+      "step": 560
+    },
+    {
+      "epoch": 1.0124444444444445,
+      "grad_norm": 1.0810185670852661,
+      "learning_rate": 0.0003524229074889868,
+      "loss": 1.8437,
+      "mean_token_accuracy": 0.6523516507525193,
+      "num_tokens": 1281232.0,
+      "step": 570
+    },
+    {
+      "epoch": 1.0302222222222222,
+      "grad_norm": 1.2796133756637573,
+      "learning_rate": 0.0003492762743864066,
+      "loss": 1.7348,
+      "mean_token_accuracy": 0.6675047099590301,
+      "num_tokens": 1304130.0,
+      "step": 580
+    },
+    {
+      "epoch": 1.048,
+      "grad_norm": 1.1196218729019165,
+      "learning_rate": 0.0003461296412838263,
+      "loss": 1.7244,
+      "mean_token_accuracy": 0.6726136475801467,
+      "num_tokens": 1326753.0,
+      "step": 590
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "grad_norm": 1.1148934364318848,
+      "learning_rate": 0.00034298300818124604,
+      "loss": 1.7216,
+      "mean_token_accuracy": 0.6700241908431053,
+      "num_tokens": 1348985.0,
+      "step": 600
+    },
+    {
+      "epoch": 1.0657777777777777,
+      "eval_loss": 1.9544994831085205,
+      "eval_mean_token_accuracy": 0.638362931728363,
+      "eval_num_tokens": 1348985.0,
+      "eval_runtime": 30.1088,
+      "eval_samples_per_second": 33.213,
+      "eval_steps_per_second": 8.303,
+      "step": 600
+    },
+    {
+      "epoch": 1.0835555555555556,
+      "grad_norm": 1.0550259351730347,
+      "learning_rate": 0.0003398363750786658,
+      "loss": 1.734,
+      "mean_token_accuracy": 0.670471416413784,
+      "num_tokens": 1371318.0,
+      "step": 610
+    },
+    {
+      "epoch": 1.1013333333333333,
+      "grad_norm": 1.1968863010406494,
+      "learning_rate": 0.0003366897419760856,
+      "loss": 1.7819,
+      "mean_token_accuracy": 0.661465460062027,
+      "num_tokens": 1393816.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.1191111111111112,
+      "grad_norm": 1.2184885740280151,
+      "learning_rate": 0.0003335431088735054,
+      "loss": 1.769,
+      "mean_token_accuracy": 0.6628920555114746,
+      "num_tokens": 1416529.0,
+      "step": 630
+    },
+    {
+      "epoch": 1.1368888888888888,
+      "grad_norm": 1.2553062438964844,
+      "learning_rate": 0.00033039647577092515,
+      "loss": 1.7807,
+      "mean_token_accuracy": 0.6629297524690628,
+      "num_tokens": 1438970.0,
+      "step": 640
+    },
+    {
+      "epoch": 1.1546666666666667,
+      "grad_norm": 1.1857445240020752,
+      "learning_rate": 0.0003272498426683449,
+      "loss": 1.7555,
+      "mean_token_accuracy": 0.6634594619274139,
+      "num_tokens": 1461324.0,
+      "step": 650
+    },
+    {
+      "epoch": 1.1724444444444444,
+      "grad_norm": 1.2381750345230103,
+      "learning_rate": 0.0003241032095657646,
+      "loss": 1.7925,
+      "mean_token_accuracy": 0.6549596875905991,
+      "num_tokens": 1484246.0,
+      "step": 660
+    },
+    {
+      "epoch": 1.1902222222222223,
+      "grad_norm": 1.187692642211914,
+      "learning_rate": 0.0003209565764631844,
+      "loss": 1.7776,
+      "mean_token_accuracy": 0.658722423017025,
+      "num_tokens": 1507305.0,
+      "step": 670
+    },
+    {
+      "epoch": 1.208,
+      "grad_norm": 1.1748046875,
+      "learning_rate": 0.00031780994336060416,
+      "loss": 1.7355,
+      "mean_token_accuracy": 0.6706653699278832,
+      "num_tokens": 1529082.0,
+      "step": 680
+    },
+    {
+      "epoch": 1.2257777777777779,
+      "grad_norm": 1.2238432168960571,
+      "learning_rate": 0.00031466331025802394,
+      "loss": 1.7462,
+      "mean_token_accuracy": 0.6667303949594497,
+      "num_tokens": 1551159.0,
+      "step": 690
+    },
+    {
+      "epoch": 1.2435555555555555,
+      "grad_norm": 1.1904925107955933,
+      "learning_rate": 0.00031151667715544366,
+      "loss": 1.7627,
+      "mean_token_accuracy": 0.6670840814709663,
+      "num_tokens": 1573348.0,
+      "step": 700
+    },
+    {
+      "epoch": 1.2613333333333334,
+      "grad_norm": 1.282875895500183,
+      "learning_rate": 0.00030837004405286344,
+      "loss": 1.7625,
+      "mean_token_accuracy": 0.6660559684038162,
+      "num_tokens": 1595989.0,
+      "step": 710
+    },
+    {
+      "epoch": 1.279111111111111,
+      "grad_norm": 1.2079155445098877,
+      "learning_rate": 0.0003052234109502832,
+      "loss": 1.7979,
+      "mean_token_accuracy": 0.6636087343096733,
+      "num_tokens": 1617895.0,
+      "step": 720
+    },
+    {
+      "epoch": 1.2968888888888888,
+      "grad_norm": 1.1903313398361206,
+      "learning_rate": 0.000302076777847703,
+      "loss": 1.7552,
+      "mean_token_accuracy": 0.6641585499048233,
+      "num_tokens": 1640415.0,
+      "step": 730
+    },
+    {
+      "epoch": 1.3146666666666667,
+      "grad_norm": 1.1872097253799438,
+      "learning_rate": 0.0002989301447451227,
+      "loss": 1.74,
+      "mean_token_accuracy": 0.6656632766127586,
+      "num_tokens": 1662713.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.3324444444444445,
+      "grad_norm": 1.1994388103485107,
+      "learning_rate": 0.00029578351164254244,
+      "loss": 1.7757,
+      "mean_token_accuracy": 0.6637446776032447,
+      "num_tokens": 1684783.0,
+      "step": 750
+    },
+    {
+      "epoch": 1.3502222222222222,
+      "grad_norm": 1.2072243690490723,
+      "learning_rate": 0.0002926368785399622,
+      "loss": 1.7702,
+      "mean_token_accuracy": 0.6637947261333466,
+      "num_tokens": 1706623.0,
+      "step": 760
+    },
+    {
+      "epoch": 1.3679999999999999,
+      "grad_norm": 1.2028874158859253,
+      "learning_rate": 0.000289490245437382,
+      "loss": 1.7952,
+      "mean_token_accuracy": 0.6561207175254822,
+      "num_tokens": 1729494.0,
+      "step": 770
+    },
+    {
+      "epoch": 1.3857777777777778,
+      "grad_norm": 1.2276169061660767,
+      "learning_rate": 0.0002863436123348018,
+      "loss": 1.7624,
+      "mean_token_accuracy": 0.664546848833561,
+      "num_tokens": 1752045.0,
+      "step": 780
+    },
+    {
+      "epoch": 1.4035555555555557,
+      "grad_norm": 1.1520951986312866,
+      "learning_rate": 0.00028319697923222156,
+      "loss": 1.7743,
+      "mean_token_accuracy": 0.6641672790050507,
+      "num_tokens": 1775601.0,
+      "step": 790
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "grad_norm": 1.219901204109192,
+      "learning_rate": 0.0002800503461296413,
+      "loss": 1.7346,
+      "mean_token_accuracy": 0.6647952407598495,
+      "num_tokens": 1798633.0,
+      "step": 800
+    },
+    {
+      "epoch": 1.4213333333333333,
+      "eval_loss": 1.9113014936447144,
+      "eval_mean_token_accuracy": 0.6443024153709411,
+      "eval_num_tokens": 1798633.0,
+      "eval_runtime": 30.1829,
+      "eval_samples_per_second": 33.131,
+      "eval_steps_per_second": 8.283,
+      "step": 800
+    },
+    {
+      "epoch": 1.439111111111111,
+      "grad_norm": 1.2413419485092163,
+      "learning_rate": 0.00027690371302706106,
+      "loss": 1.7157,
+      "mean_token_accuracy": 0.6692047983407974,
+      "num_tokens": 1820656.0,
+      "step": 810
+    },
+    {
+      "epoch": 1.456888888888889,
+      "grad_norm": 1.2917033433914185,
+      "learning_rate": 0.0002737570799244808,
+      "loss": 1.7631,
+      "mean_token_accuracy": 0.6633412629365921,
+      "num_tokens": 1842871.0,
+      "step": 820
+    },
+    {
+      "epoch": 1.4746666666666668,
+      "grad_norm": 1.2017630338668823,
+      "learning_rate": 0.00027061044682190056,
+      "loss": 1.726,
+      "mean_token_accuracy": 0.6701689288020134,
+      "num_tokens": 1865652.0,
+      "step": 830
+    },
+    {
+      "epoch": 1.4924444444444445,
+      "grad_norm": 1.1332948207855225,
+      "learning_rate": 0.00026746381371932034,
+      "loss": 1.7561,
+      "mean_token_accuracy": 0.6648476481437683,
+      "num_tokens": 1888455.0,
+      "step": 840
+    },
+    {
+      "epoch": 1.5102222222222221,
+      "grad_norm": 1.2592897415161133,
+      "learning_rate": 0.00026431718061674007,
+      "loss": 1.7194,
+      "mean_token_accuracy": 0.6733420416712761,
+      "num_tokens": 1910676.0,
+      "step": 850
+    },
+    {
+      "epoch": 1.528,
+      "grad_norm": 1.2740918397903442,
+      "learning_rate": 0.00026117054751415985,
+      "loss": 1.7502,
+      "mean_token_accuracy": 0.6677413523197174,
+      "num_tokens": 1933253.0,
+      "step": 860
+    },
+    {
+      "epoch": 1.545777777777778,
+      "grad_norm": 1.2723543643951416,
+      "learning_rate": 0.0002580239144115796,
+      "loss": 1.7622,
+      "mean_token_accuracy": 0.6622136697173119,
+      "num_tokens": 1955820.0,
+      "step": 870
+    },
+    {
+      "epoch": 1.5635555555555556,
+      "grad_norm": 1.2220773696899414,
+      "learning_rate": 0.0002548772813089994,
+      "loss": 1.7568,
+      "mean_token_accuracy": 0.6640823170542717,
+      "num_tokens": 1978120.0,
+      "step": 880
+    },
+    {
+      "epoch": 1.5813333333333333,
+      "grad_norm": 1.2161861658096313,
+      "learning_rate": 0.0002517306482064191,
+      "loss": 1.7031,
+      "mean_token_accuracy": 0.6733875796198845,
+      "num_tokens": 2000508.0,
+      "step": 890
+    },
+    {
+      "epoch": 1.5991111111111111,
+      "grad_norm": 1.1472899913787842,
+      "learning_rate": 0.0002485840151038389,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6644304901361465,
+      "num_tokens": 2023356.0,
+      "step": 900
+    },
+    {
+      "epoch": 1.616888888888889,
+      "grad_norm": 1.177524447441101,
+      "learning_rate": 0.00024543738200125863,
+      "loss": 1.6951,
+      "mean_token_accuracy": 0.6781735569238663,
+      "num_tokens": 2045965.0,
+      "step": 910
+    },
+    {
+      "epoch": 1.6346666666666667,
+      "grad_norm": 1.158720850944519,
+      "learning_rate": 0.0002422907488986784,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6616655588150024,
+      "num_tokens": 2069108.0,
+      "step": 920
+    },
+    {
+      "epoch": 1.6524444444444444,
+      "grad_norm": 1.3048863410949707,
+      "learning_rate": 0.0002391441157960982,
+      "loss": 1.7596,
+      "mean_token_accuracy": 0.6625866696238518,
+      "num_tokens": 2091755.0,
+      "step": 930
+    },
+    {
+      "epoch": 1.6702222222222223,
+      "grad_norm": 1.3040592670440674,
+      "learning_rate": 0.00023599748269351794,
+      "loss": 1.7286,
+      "mean_token_accuracy": 0.6688349068164825,
+      "num_tokens": 2114496.0,
+      "step": 940
+    },
+    {
+      "epoch": 1.688,
+      "grad_norm": 1.1945664882659912,
+      "learning_rate": 0.0002328508495909377,
+      "loss": 1.7479,
+      "mean_token_accuracy": 0.6634002208709717,
+      "num_tokens": 2137041.0,
+      "step": 950
+    },
+    {
+      "epoch": 1.7057777777777776,
+      "grad_norm": 1.2552154064178467,
+      "learning_rate": 0.00022970421648835747,
+      "loss": 1.7537,
+      "mean_token_accuracy": 0.6634084343910217,
+      "num_tokens": 2159711.0,
+      "step": 960
+    },
+    {
+      "epoch": 1.7235555555555555,
+      "grad_norm": 1.2484403848648071,
+      "learning_rate": 0.00022655758338577722,
+      "loss": 1.7557,
+      "mean_token_accuracy": 0.6687701687216758,
+      "num_tokens": 2181675.0,
+      "step": 970
+    },
+    {
+      "epoch": 1.7413333333333334,
+      "grad_norm": 1.1729741096496582,
+      "learning_rate": 0.000223410950283197,
+      "loss": 1.7274,
+      "mean_token_accuracy": 0.6730923235416413,
+      "num_tokens": 2204671.0,
+      "step": 980
+    },
+    {
+      "epoch": 1.759111111111111,
+      "grad_norm": 1.312406301498413,
+      "learning_rate": 0.00022026431718061672,
+      "loss": 1.7377,
+      "mean_token_accuracy": 0.6612286046147346,
+      "num_tokens": 2227866.0,
+      "step": 990
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "grad_norm": 1.3160700798034668,
+      "learning_rate": 0.0002171176840780365,
+      "loss": 1.7224,
+      "mean_token_accuracy": 0.6710903704166412,
+      "num_tokens": 2250379.0,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7768888888888887,
+      "eval_loss": 1.8698750734329224,
+      "eval_mean_token_accuracy": 0.6504300956726075,
+      "eval_num_tokens": 2250379.0,
+      "eval_runtime": 30.1651,
+      "eval_samples_per_second": 33.151,
+      "eval_steps_per_second": 8.288,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7946666666666666,
+      "grad_norm": 1.1682268381118774,
+      "learning_rate": 0.00021397105097545628,
+      "loss": 1.7458,
+      "mean_token_accuracy": 0.665731918811798,
+      "num_tokens": 2272999.0,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8124444444444445,
+      "grad_norm": 1.1300792694091797,
+      "learning_rate": 0.00021082441787287603,
+      "loss": 1.7261,
+      "mean_token_accuracy": 0.6688766568899155,
+      "num_tokens": 2295110.0,
+      "step": 1020
+    },
+    {
+      "epoch": 1.8302222222222222,
+      "grad_norm": 1.2450717687606812,
+      "learning_rate": 0.00020767778477029578,
+      "loss": 1.6968,
+      "mean_token_accuracy": 0.6718593999743462,
+      "num_tokens": 2317432.0,
+      "step": 1030
+    },
+    {
+      "epoch": 1.8479999999999999,
+      "grad_norm": 1.201785683631897,
+      "learning_rate": 0.00020453115166771556,
+      "loss": 1.7411,
+      "mean_token_accuracy": 0.6636266440153122,
+      "num_tokens": 2340305.0,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8657777777777778,
+      "grad_norm": 1.2161189317703247,
+      "learning_rate": 0.0002013845185651353,
+      "loss": 1.7335,
+      "mean_token_accuracy": 0.6697820112109184,
+      "num_tokens": 2362746.0,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8835555555555556,
+      "grad_norm": 1.209251046180725,
+      "learning_rate": 0.00019823788546255506,
+      "loss": 1.7197,
+      "mean_token_accuracy": 0.6663407534360886,
+      "num_tokens": 2385647.0,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9013333333333333,
+      "grad_norm": 1.1715242862701416,
+      "learning_rate": 0.00019509125235997482,
+      "loss": 1.7298,
+      "mean_token_accuracy": 0.6650380924344063,
+      "num_tokens": 2408732.0,
+      "step": 1070
+    },
+    {
+      "epoch": 1.919111111111111,
+      "grad_norm": 1.2717958688735962,
+      "learning_rate": 0.0001919446192573946,
+      "loss": 1.7101,
+      "mean_token_accuracy": 0.672779667377472,
+      "num_tokens": 2431137.0,
+      "step": 1080
+    },
+    {
+      "epoch": 1.9368888888888889,
+      "grad_norm": 1.2264835834503174,
+      "learning_rate": 0.00018879798615481437,
+      "loss": 1.6754,
+      "mean_token_accuracy": 0.6768045455217362,
+      "num_tokens": 2453517.0,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9546666666666668,
+      "grad_norm": 1.3289543390274048,
+      "learning_rate": 0.0001856513530522341,
+      "loss": 1.7178,
+      "mean_token_accuracy": 0.6685071781277656,
+      "num_tokens": 2475737.0,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9724444444444444,
+      "grad_norm": 1.1367521286010742,
+      "learning_rate": 0.00018250471994965388,
+      "loss": 1.7109,
+      "mean_token_accuracy": 0.6677573248744011,
+      "num_tokens": 2497880.0,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9902222222222221,
+      "grad_norm": 1.246870994567871,
+      "learning_rate": 0.00017935808684707363,
+      "loss": 1.7317,
+      "mean_token_accuracy": 0.6695448830723763,
+      "num_tokens": 2519974.0,
+      "step": 1120
+    },
+    {
+      "epoch": 2.007111111111111,
+      "grad_norm": 1.0757529735565186,
+      "learning_rate": 0.0001762114537444934,
+      "loss": 1.586,
+      "mean_token_accuracy": 0.6983135671992051,
+      "num_tokens": 2540296.0,
+      "step": 1130
+    },
+    {
+      "epoch": 2.024888888888889,
+      "grad_norm": 1.3501553535461426,
+      "learning_rate": 0.00017306482064191316,
+      "loss": 1.4761,
+      "mean_token_accuracy": 0.7080946862697601,
+      "num_tokens": 2561919.0,
+      "step": 1140
+    },
+    {
+      "epoch": 2.042666666666667,
+      "grad_norm": 1.502793788909912,
+      "learning_rate": 0.0001699181875393329,
+      "loss": 1.5049,
+      "mean_token_accuracy": 0.7034129843115806,
+      "num_tokens": 2584915.0,
+      "step": 1150
+    },
+    {
+      "epoch": 2.0604444444444443,
+      "grad_norm": 1.2668856382369995,
+      "learning_rate": 0.0001667715544367527,
+      "loss": 1.4612,
+      "mean_token_accuracy": 0.7100765943527222,
+      "num_tokens": 2606857.0,
+      "step": 1160
+    },
+    {
+      "epoch": 2.078222222222222,
+      "grad_norm": 1.386775016784668,
+      "learning_rate": 0.00016362492133417244,
+      "loss": 1.4695,
+      "mean_token_accuracy": 0.7058908969163895,
+      "num_tokens": 2629737.0,
+      "step": 1170
+    },
+    {
+      "epoch": 2.096,
+      "grad_norm": 1.3320142030715942,
+      "learning_rate": 0.0001604782882315922,
+      "loss": 1.4337,
+      "mean_token_accuracy": 0.7136739388108253,
+      "num_tokens": 2651936.0,
+      "step": 1180
+    },
+    {
+      "epoch": 2.113777777777778,
+      "grad_norm": 1.289437174797058,
+      "learning_rate": 0.00015733165512901197,
+      "loss": 1.4975,
+      "mean_token_accuracy": 0.7037445783615113,
+      "num_tokens": 2675241.0,
+      "step": 1190
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "grad_norm": 1.208194613456726,
+      "learning_rate": 0.00015418502202643172,
+      "loss": 1.4769,
+      "mean_token_accuracy": 0.7093130722641945,
+      "num_tokens": 2698219.0,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1315555555555554,
+      "eval_loss": 1.8875290155410767,
+      "eval_mean_token_accuracy": 0.6501186113357544,
+      "eval_num_tokens": 2698219.0,
+      "eval_runtime": 30.175,
+      "eval_samples_per_second": 33.14,
+      "eval_steps_per_second": 8.285,
+      "step": 1200
+    },
+    {
+      "epoch": 2.1493333333333333,
+      "grad_norm": 1.360378623008728,
+      "learning_rate": 0.0001510383889238515,
+      "loss": 1.485,
+      "mean_token_accuracy": 0.7059277340769767,
+      "num_tokens": 2721580.0,
+      "step": 1210
+    },
+    {
+      "epoch": 2.167111111111111,
+      "grad_norm": 1.3289263248443604,
+      "learning_rate": 0.00014789175582127122,
+      "loss": 1.4869,
+      "mean_token_accuracy": 0.7061908945441246,
+      "num_tokens": 2744110.0,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1848888888888887,
+      "grad_norm": 1.2693774700164795,
+      "learning_rate": 0.000144745122718691,
+      "loss": 1.4555,
+      "mean_token_accuracy": 0.712950111925602,
+      "num_tokens": 2765844.0,
+      "step": 1230
+    },
+    {
+      "epoch": 2.2026666666666666,
+      "grad_norm": 1.3220160007476807,
+      "learning_rate": 0.00014159848961611078,
+      "loss": 1.4634,
+      "mean_token_accuracy": 0.7090483531355858,
+      "num_tokens": 2788070.0,
+      "step": 1240
+    },
+    {
+      "epoch": 2.2204444444444444,
+      "grad_norm": 1.2682182788848877,
+      "learning_rate": 0.00013845185651353053,
+      "loss": 1.497,
+      "mean_token_accuracy": 0.7039189741015435,
+      "num_tokens": 2810964.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.2382222222222223,
+      "grad_norm": 1.3531242609024048,
+      "learning_rate": 0.00013530522341095028,
+      "loss": 1.4625,
+      "mean_token_accuracy": 0.7105821236968041,
+      "num_tokens": 2833176.0,
+      "step": 1260
+    },
+    {
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.2819761037826538,
+      "learning_rate": 0.00013215859030837003,
+      "loss": 1.5092,
+      "mean_token_accuracy": 0.7021819025278091,
+      "num_tokens": 2855440.0,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2737777777777777,
+      "grad_norm": 1.3131153583526611,
+      "learning_rate": 0.0001290119572057898,
+      "loss": 1.4458,
+      "mean_token_accuracy": 0.7121976673603058,
+      "num_tokens": 2877713.0,
+      "step": 1280
+    },
+    {
+      "epoch": 2.2915555555555556,
+      "grad_norm": 1.2332444190979004,
+      "learning_rate": 0.00012586532410320956,
+      "loss": 1.4797,
+      "mean_token_accuracy": 0.7103414624929428,
+      "num_tokens": 2899934.0,
+      "step": 1290
+    },
+    {
+      "epoch": 2.3093333333333335,
+      "grad_norm": 1.2947789430618286,
+      "learning_rate": 0.00012271869100062931,
+      "loss": 1.4969,
+      "mean_token_accuracy": 0.7032593175768852,
+      "num_tokens": 2923244.0,
+      "step": 1300
+    },
+    {
+      "epoch": 2.327111111111111,
+      "grad_norm": 1.4002538919448853,
+      "learning_rate": 0.0001195720578980491,
+      "loss": 1.4538,
+      "mean_token_accuracy": 0.711962579190731,
+      "num_tokens": 2945580.0,
+      "step": 1310
+    },
+    {
+      "epoch": 2.344888888888889,
+      "grad_norm": 1.270635962486267,
+      "learning_rate": 0.00011642542479546884,
+      "loss": 1.4622,
+      "mean_token_accuracy": 0.7059817537665367,
+      "num_tokens": 2969470.0,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3626666666666667,
+      "grad_norm": 1.3131049871444702,
+      "learning_rate": 0.00011327879169288861,
+      "loss": 1.4597,
+      "mean_token_accuracy": 0.7036674708127976,
+      "num_tokens": 2991982.0,
+      "step": 1330
+    },
+    {
+      "epoch": 2.3804444444444446,
+      "grad_norm": 1.3708815574645996,
+      "learning_rate": 0.00011013215859030836,
+      "loss": 1.4837,
+      "mean_token_accuracy": 0.706661482155323,
+      "num_tokens": 3014206.0,
+      "step": 1340
+    },
+    {
+      "epoch": 2.398222222222222,
+      "grad_norm": 1.3198243379592896,
+      "learning_rate": 0.00010698552548772814,
+      "loss": 1.4431,
+      "mean_token_accuracy": 0.7101009368896485,
+      "num_tokens": 3036651.0,
+      "step": 1350
+    },
+    {
+      "epoch": 2.416,
+      "grad_norm": 1.280879259109497,
+      "learning_rate": 0.00010383889238514789,
+      "loss": 1.477,
+      "mean_token_accuracy": 0.7085638150572777,
+      "num_tokens": 3058745.0,
+      "step": 1360
+    },
+    {
+      "epoch": 2.433777777777778,
+      "grad_norm": 1.419290542602539,
+      "learning_rate": 0.00010069225928256766,
+      "loss": 1.4887,
+      "mean_token_accuracy": 0.7048677980899811,
+      "num_tokens": 3081726.0,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4515555555555557,
+      "grad_norm": 1.3684442043304443,
+      "learning_rate": 9.754562617998741e-05,
+      "loss": 1.4677,
+      "mean_token_accuracy": 0.7087965488433838,
+      "num_tokens": 3104292.0,
+      "step": 1380
+    },
+    {
+      "epoch": 2.469333333333333,
+      "grad_norm": 1.3453881740570068,
+      "learning_rate": 9.439899307740719e-05,
+      "loss": 1.4682,
+      "mean_token_accuracy": 0.7076600238680839,
+      "num_tokens": 3126334.0,
+      "step": 1390
+    },
+    {
+      "epoch": 2.487111111111111,
+      "grad_norm": 1.2759828567504883,
+      "learning_rate": 9.125235997482694e-05,
+      "loss": 1.4971,
+      "mean_token_accuracy": 0.7039492294192314,
+      "num_tokens": 3149845.0,
+      "step": 1400
+    },
+    {
+      "epoch": 2.487111111111111,
+      "eval_loss": 1.873150110244751,
+      "eval_mean_token_accuracy": 0.65266099858284,
+      "eval_num_tokens": 3149845.0,
+      "eval_runtime": 30.0464,
+      "eval_samples_per_second": 33.282,
+      "eval_steps_per_second": 8.32,
+      "step": 1400
+    },
+    {
+      "epoch": 2.504888888888889,
+      "grad_norm": 1.3677406311035156,
+      "learning_rate": 8.81057268722467e-05,
+      "loss": 1.4905,
+      "mean_token_accuracy": 0.7065914765000343,
+      "num_tokens": 3172117.0,
+      "step": 1410
+    },
+    {
+      "epoch": 2.522666666666667,
+      "grad_norm": 1.2730094194412231,
+      "learning_rate": 8.495909376966645e-05,
+      "loss": 1.4772,
+      "mean_token_accuracy": 0.7116217255592346,
+      "num_tokens": 3194261.0,
+      "step": 1420
+    },
+    {
+      "epoch": 2.5404444444444443,
+      "grad_norm": 1.3448493480682373,
+      "learning_rate": 8.181246066708622e-05,
+      "loss": 1.3895,
+      "mean_token_accuracy": 0.7193023800849915,
+      "num_tokens": 3216455.0,
+      "step": 1430
+    },
+    {
+      "epoch": 2.558222222222222,
+      "grad_norm": 1.3793234825134277,
+      "learning_rate": 7.866582756450598e-05,
+      "loss": 1.4958,
+      "mean_token_accuracy": 0.7059502750635147,
+      "num_tokens": 3238359.0,
+      "step": 1440
+    },
+    {
+      "epoch": 2.576,
+      "grad_norm": 1.3225526809692383,
+      "learning_rate": 7.551919446192575e-05,
+      "loss": 1.5047,
+      "mean_token_accuracy": 0.7010831996798516,
+      "num_tokens": 3261568.0,
+      "step": 1450
+    },
+    {
+      "epoch": 2.5937777777777775,
+      "grad_norm": 1.326993465423584,
+      "learning_rate": 7.23725613593455e-05,
+      "loss": 1.5066,
+      "mean_token_accuracy": 0.6991274550557136,
+      "num_tokens": 3284517.0,
+      "step": 1460
+    },
+    {
+      "epoch": 2.6115555555555554,
+      "grad_norm": 1.3944668769836426,
+      "learning_rate": 6.922592825676527e-05,
+      "loss": 1.461,
+      "mean_token_accuracy": 0.7067686557769776,
+      "num_tokens": 3306623.0,
+      "step": 1470
+    },
+    {
+      "epoch": 2.6293333333333333,
+      "grad_norm": 1.2688894271850586,
+      "learning_rate": 6.607929515418502e-05,
+      "loss": 1.4464,
+      "mean_token_accuracy": 0.7143596678972244,
+      "num_tokens": 3329270.0,
+      "step": 1480
+    },
+    {
+      "epoch": 2.647111111111111,
+      "grad_norm": 1.245979905128479,
+      "learning_rate": 6.293266205160478e-05,
+      "loss": 1.4745,
+      "mean_token_accuracy": 0.7083328410983085,
+      "num_tokens": 3352533.0,
+      "step": 1490
+    },
+    {
+      "epoch": 2.664888888888889,
+      "grad_norm": 1.359151840209961,
+      "learning_rate": 5.978602894902455e-05,
+      "loss": 1.4547,
+      "mean_token_accuracy": 0.7116006925702095,
+      "num_tokens": 3374132.0,
+      "step": 1500
+    },
+    {
+      "epoch": 2.6826666666666665,
+      "grad_norm": 1.3699895143508911,
+      "learning_rate": 5.6639395846444305e-05,
+      "loss": 1.4876,
+      "mean_token_accuracy": 0.702751100063324,
+      "num_tokens": 3397176.0,
+      "step": 1510
+    },
+    {
+      "epoch": 2.7004444444444444,
+      "grad_norm": 1.3784525394439697,
+      "learning_rate": 5.349276274386407e-05,
+      "loss": 1.4384,
+      "mean_token_accuracy": 0.713467463850975,
+      "num_tokens": 3418961.0,
+      "step": 1520
+    },
+    {
+      "epoch": 2.7182222222222223,
+      "grad_norm": 1.3480818271636963,
+      "learning_rate": 5.034612964128383e-05,
+      "loss": 1.4436,
+      "mean_token_accuracy": 0.7122948378324508,
+      "num_tokens": 3441815.0,
+      "step": 1530
+    },
+    {
+      "epoch": 2.7359999999999998,
+      "grad_norm": 1.2246533632278442,
+      "learning_rate": 4.719949653870359e-05,
+      "loss": 1.4695,
+      "mean_token_accuracy": 0.70788504332304,
+      "num_tokens": 3464582.0,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7537777777777777,
+      "grad_norm": 1.2388625144958496,
+      "learning_rate": 4.405286343612335e-05,
+      "loss": 1.433,
+      "mean_token_accuracy": 0.7149359166622162,
+      "num_tokens": 3486694.0,
+      "step": 1550
+    },
+    {
+      "epoch": 2.7715555555555556,
+      "grad_norm": 1.317224383354187,
+      "learning_rate": 4.090623033354311e-05,
+      "loss": 1.4772,
+      "mean_token_accuracy": 0.7114273443818092,
+      "num_tokens": 3508908.0,
+      "step": 1560
+    },
+    {
+      "epoch": 2.7893333333333334,
+      "grad_norm": 1.2926599979400635,
+      "learning_rate": 3.7759597230962874e-05,
+      "loss": 1.425,
+      "mean_token_accuracy": 0.7173908859491348,
+      "num_tokens": 3530306.0,
+      "step": 1570
+    },
+    {
+      "epoch": 2.8071111111111113,
+      "grad_norm": 1.3763422966003418,
+      "learning_rate": 3.461296412838263e-05,
+      "loss": 1.4996,
+      "mean_token_accuracy": 0.7037103369832038,
+      "num_tokens": 3552880.0,
+      "step": 1580
+    },
+    {
+      "epoch": 2.824888888888889,
+      "grad_norm": 1.3559826612472534,
+      "learning_rate": 3.146633102580239e-05,
+      "loss": 1.4756,
+      "mean_token_accuracy": 0.7092328786849975,
+      "num_tokens": 3575800.0,
+      "step": 1590
+    },
+    {
+      "epoch": 2.8426666666666667,
+      "grad_norm": 1.3408883810043335,
+      "learning_rate": 2.8319697923222152e-05,
+      "loss": 1.4735,
+      "mean_token_accuracy": 0.7120713770389557,
+      "num_tokens": 3598214.0,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8426666666666667,
+      "eval_loss": 1.8612734079360962,
+      "eval_mean_token_accuracy": 0.6546365599632263,
+      "eval_num_tokens": 3598214.0,
+      "eval_runtime": 30.1859,
+      "eval_samples_per_second": 33.128,
+      "eval_steps_per_second": 8.282,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8604444444444446,
+      "grad_norm": 1.2175135612487793,
+      "learning_rate": 2.5173064820641914e-05,
+      "loss": 1.454,
+      "mean_token_accuracy": 0.7124782979488373,
+      "num_tokens": 3620353.0,
+      "step": 1610
+    },
+    {
+      "epoch": 2.878222222222222,
+      "grad_norm": 1.2944847345352173,
+      "learning_rate": 2.2026431718061676e-05,
+      "loss": 1.4631,
+      "mean_token_accuracy": 0.7102750897407532,
+      "num_tokens": 3642281.0,
+      "step": 1620
+    },
+    {
+      "epoch": 2.896,
+      "grad_norm": 1.2475266456604004,
+      "learning_rate": 1.8879798615481437e-05,
+      "loss": 1.4233,
+      "mean_token_accuracy": 0.7155472055077553,
+      "num_tokens": 3664853.0,
+      "step": 1630
+    },
+    {
+      "epoch": 2.913777777777778,
+      "grad_norm": 1.3260661363601685,
+      "learning_rate": 1.5733165512901195e-05,
+      "loss": 1.5218,
+      "mean_token_accuracy": 0.7012277022004128,
+      "num_tokens": 3688123.0,
+      "step": 1640
+    },
+    {
+      "epoch": 2.9315555555555557,
+      "grad_norm": 1.3332605361938477,
+      "learning_rate": 1.2586532410320957e-05,
+      "loss": 1.5025,
+      "mean_token_accuracy": 0.7005213543772697,
+      "num_tokens": 3710934.0,
+      "step": 1650
+    },
+    {
+      "epoch": 2.9493333333333336,
+      "grad_norm": 1.3807092905044556,
+      "learning_rate": 9.439899307740719e-06,
+      "loss": 1.5066,
+      "mean_token_accuracy": 0.7041444540023803,
+      "num_tokens": 3733298.0,
+      "step": 1660
+    },
+    {
+      "epoch": 2.967111111111111,
+      "grad_norm": 1.263149619102478,
+      "learning_rate": 6.2932662051604785e-06,
+      "loss": 1.5033,
+      "mean_token_accuracy": 0.7054026111960411,
+      "num_tokens": 3755641.0,
+      "step": 1670
+    },
+    {
+      "epoch": 2.984888888888889,
+      "grad_norm": 1.3899400234222412,
+      "learning_rate": 3.1466331025802393e-06,
+      "loss": 1.4425,
+      "mean_token_accuracy": 0.7106350988149643,
+      "num_tokens": 3777938.0,
+      "step": 1680
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1689,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3370037644550144e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

special_tokens_map.json

@@ -0,0 +1,86 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endofturn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,501 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|_unuse_missing_100256|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|_unuse_missing_100261|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|_unuse_missing_100262|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|_unuse_missing_100263|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|_unuse_missing_100264|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|_unuse_missing_100265|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|_unuse_missing_100266|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|_unuse_missing_100267|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|_unuse_missing_100268|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|_unuse_missing_100269|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|_unuse_missing_100270|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|_unuse_missing_100271|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|stop|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|endofturn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110491": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110492": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110493": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110494": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110495": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110496": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110497": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110498": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110499": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110500": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110501": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110502": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110503": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110504": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110505": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110506": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110507": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110508": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110509": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110510": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110511": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110512": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110513": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110514": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110515": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110516": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110517": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110518": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110519": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110520": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110521": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110522": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110523": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|endofprompt|>",
+    "<|_unuse_missing_100256|>",
+    "<|_unuse_missing_100261|>",
+    "<|_unuse_missing_100262|>",
+    "<|_unuse_missing_100263|>",
+    "<|_unuse_missing_100264|>",
+    "<|_unuse_missing_100265|>",
+    "<|_unuse_missing_100266|>",
+    "<|_unuse_missing_100267|>",
+    "<|_unuse_missing_100268|>",
+    "<|_unuse_missing_100269|>",
+    "<|_unuse_missing_100270|>",
+    "<|_unuse_missing_100271|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|stop|>",
+    "<|endofturn|>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endofturn|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd15c2ce737a7ea199ddf9399d7f90d562a3ce4f7e1a07c04ec4267cbad52d6d
+size 5624