checkpoint step 7000

checkpoint-6700/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Dorn4449/CyberSentinel-Mistral-7B-v3.8
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

checkpoint-6700/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6700/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a31b1767dedc516354c90ae65c6652b4b33f3086a42e32da86b4aa9d997f3615
+size 27297032

checkpoint-6700/checkpoint-6000/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6700/checkpoint-6000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:286625e19ea92f6bcbc715bf5d40bac2ac85da8b9ecb31679d50c38cb4b4b694
+size 27297032

checkpoint-6700/checkpoint-6100/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6700/checkpoint-6100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a6ee2942c85113263a2331006fa30db088217df193b0fca3de2eebe282399b
+size 27297032

checkpoint-6700/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71c4e836dc5556b91ee2c11ccbe88fce50624eebd3fa0004f286582515d980b2
+size 54744314

checkpoint-6700/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be66c1af761364c52f99e5a1734b6bc5a54ba1e66ffb256f82e96523353ef894
+size 14244

checkpoint-6700/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc6199eef99b9032101aaf24b46c5cbd4bbd3bcbedab26fe5c2688d336199299
+size 1064

checkpoint-6700/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-6700/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

checkpoint-6700/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-6700/trainer_state.json

@@ -0,0 +1,679 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.17866666666666667,
+  "eval_steps": 500,
+  "global_step": 6700,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.084044404476881,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 0.796875,
+      "learning_rate": 7.92e-07,
+      "loss": 1.3782466125488282,
+      "mean_token_accuracy": 0.7083857330679894,
+      "num_tokens": 986491.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1007767178118228,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 0.3125,
+      "learning_rate": 1.592e-06,
+      "loss": 1.2902149963378906,
+      "mean_token_accuracy": 0.7171274860203266,
+      "num_tokens": 1952567.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0802835547924041,
+      "epoch": 0.032,
+      "grad_norm": 0.28515625,
+      "learning_rate": 2.392e-06,
+      "loss": 1.170371322631836,
+      "mean_token_accuracy": 0.7354862231016159,
+      "num_tokens": 2935479.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0573877203464508,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 0.23046875,
+      "learning_rate": 3.192e-06,
+      "loss": 1.062843780517578,
+      "mean_token_accuracy": 0.7536254300177098,
+      "num_tokens": 3920344.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0149462178349495,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.212890625,
+      "learning_rate": 3.992e-06,
+      "loss": 0.9590372467041015,
+      "mean_token_accuracy": 0.7738636130094528,
+      "num_tokens": 4899217.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.8931056271493435,
+      "epoch": 0.064,
+      "grad_norm": 0.28125,
+      "learning_rate": 4.792e-06,
+      "loss": 0.7998863983154297,
+      "mean_token_accuracy": 0.8012332341074944,
+      "num_tokens": 5883391.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.6136953190714121,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 1.1015625,
+      "learning_rate": 5.592000000000001e-06,
+      "loss": 0.5131131362915039,
+      "mean_token_accuracy": 0.866335421949625,
+      "num_tokens": 6857125.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.2726459547691047,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.474609375,
+      "learning_rate": 6.392e-06,
+      "loss": 0.18439870834350586,
+      "mean_token_accuracy": 0.9579810079932213,
+      "num_tokens": 7831290.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.11284050881862641,
+      "epoch": 0.096,
+      "grad_norm": 0.1875,
+      "learning_rate": 7.192e-06,
+      "loss": 0.05271556854248047,
+      "mean_token_accuracy": 0.9904264670610428,
+      "num_tokens": 8815417.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.068436967888847,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.125,
+      "learning_rate": 7.992e-06,
+      "loss": 0.02549468755722046,
+      "mean_token_accuracy": 0.9951217715442181,
+      "num_tokens": 9782033.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.052372096767649055,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 8.792e-06,
+      "loss": 0.017259199619293213,
+      "mean_token_accuracy": 0.9962555834650993,
+      "num_tokens": 10759962.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.03862797610927373,
+      "epoch": 0.128,
+      "grad_norm": 0.0867946669459343,
+      "learning_rate": 9.591999999999999e-06,
+      "loss": 0.013324768543243408,
+      "mean_token_accuracy": 0.9968111206591129,
+      "num_tokens": 987986.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.032093781144358215,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 0.037592533975839615,
+      "learning_rate": 1.0392e-05,
+      "loss": 0.013392001390457153,
+      "mean_token_accuracy": 0.9965760576725006,
+      "num_tokens": 1968127.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.031904329673852774,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 0.10515860468149185,
+      "learning_rate": 1.1192e-05,
+      "loss": 0.016373103857040404,
+      "mean_token_accuracy": 0.9955054900050163,
+      "num_tokens": 2965677.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.0265167937008664,
+      "epoch": 0.16,
+      "grad_norm": 0.12689532339572906,
+      "learning_rate": 1.1992e-05,
+      "loss": 0.011554093360900878,
+      "mean_token_accuracy": 0.996508517563343,
+      "num_tokens": 3938144.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.024146563813555986,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.03401608020067215,
+      "learning_rate": 1.2792e-05,
+      "loss": 0.010398292541503906,
+      "mean_token_accuracy": 0.9966818282008171,
+      "num_tokens": 4897019.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.025755486716516316,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.05834396556019783,
+      "learning_rate": 1.3592000000000001e-05,
+      "loss": 0.012847075462341309,
+      "mean_token_accuracy": 0.9960671140253544,
+      "num_tokens": 5880125.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.023504739217460154,
+      "epoch": 0.192,
+      "grad_norm": 0.09117468446493149,
+      "learning_rate": 1.4392e-05,
+      "loss": 0.011869451999664306,
+      "mean_token_accuracy": 0.9964955732226372,
+      "num_tokens": 6867360.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.022944011739455164,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 0.046192608773708344,
+      "learning_rate": 1.4978666666666668e-05,
+      "loss": 0.010620262622833252,
+      "mean_token_accuracy": 0.9964672869443894,
+      "num_tokens": 7840472.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.018894561287015676,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.02761668898165226,
+      "learning_rate": 1.4889777777777778e-05,
+      "loss": 0.0067046540975570675,
+      "mean_token_accuracy": 0.9975774252414703,
+      "num_tokens": 8827474.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.020934174589347095,
+      "epoch": 0.224,
+      "grad_norm": 0.02931295707821846,
+      "learning_rate": 1.4800888888888889e-05,
+      "loss": 0.010140993595123292,
+      "mean_token_accuracy": 0.996714953482151,
+      "num_tokens": 9808475.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.020719884738791734,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 0.02652113325893879,
+      "learning_rate": 1.4712e-05,
+      "loss": 0.010557392835617066,
+      "mean_token_accuracy": 0.9969246552884579,
+      "num_tokens": 10782725.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.020559412932489068,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 0.032906673848629,
+      "learning_rate": 1.4623111111111113e-05,
+      "loss": 0.009789772033691406,
+      "mean_token_accuracy": 0.9966975942254066,
+      "num_tokens": 11769893.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.018388252432923764,
+      "epoch": 0.256,
+      "grad_norm": 0.08162333816289902,
+      "learning_rate": 1.4534222222222222e-05,
+      "loss": 0.007909480929374695,
+      "mean_token_accuracy": 0.9972486282885075,
+      "num_tokens": 12753597.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.02204789153067395,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.023511990904808044,
+      "learning_rate": 1.4445333333333334e-05,
+      "loss": 0.011945382356643677,
+      "mean_token_accuracy": 0.9963182592391968,
+      "num_tokens": 13739044.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.02037038065260276,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 0.02014540508389473,
+      "learning_rate": 1.4356444444444446e-05,
+      "loss": 0.010165022611618042,
+      "mean_token_accuracy": 0.9967658732831478,
+      "num_tokens": 14709501.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.01947357293218374,
+      "epoch": 0.288,
+      "grad_norm": 0.01829116977751255,
+      "learning_rate": 1.4267555555555555e-05,
+      "loss": 0.009100326895713806,
+      "mean_token_accuracy": 0.9969173397123814,
+      "num_tokens": 15693309.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.021565243480727078,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 0.031732361763715744,
+      "learning_rate": 1.4178666666666667e-05,
+      "loss": 0.01107092022895813,
+      "mean_token_accuracy": 0.9963353677093982,
+      "num_tokens": 985520.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.02136099517461844,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 0.03870174661278725,
+      "learning_rate": 1.4089777777777779e-05,
+      "loss": 0.011569523811340332,
+      "mean_token_accuracy": 0.9964252272248268,
+      "num_tokens": 1969572.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.019074252294376492,
+      "epoch": 0.32,
+      "grad_norm": 0.01954864338040352,
+      "learning_rate": 1.4000888888888888e-05,
+      "loss": 0.008727578520774841,
+      "mean_token_accuracy": 0.9970971086621284,
+      "num_tokens": 2945404.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.018130726229865102,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 0.01979902759194374,
+      "learning_rate": 1.3912e-05,
+      "loss": 0.007898266315460206,
+      "mean_token_accuracy": 0.9971335357427598,
+      "num_tokens": 3925122.0,
+      "step": 3100
+    },
+    {
+      "entropy": 0.019400757937546587,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 0.030273959040641785,
+      "learning_rate": 1.0980206234443522e-05,
+      "loss": 0.009979066749413809,
+      "mean_token_accuracy": 0.9968534293584526,
+      "num_tokens": 945781.0,
+      "step": 3200
+    },
+    {
+      "entropy": 0.021496493350714446,
+      "epoch": 0.352,
+      "grad_norm": 0.023218955844640732,
+      "learning_rate": 1.0802417921061989e-05,
+      "loss": 0.011733214855194091,
+      "mean_token_accuracy": 0.996385814100504,
+      "num_tokens": 1919451.0,
+      "step": 3300
+    },
+    {
+      "entropy": 0.01932728004641831,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 0.014179096557199955,
+      "learning_rate": 1.0624629607680455e-05,
+      "loss": 0.00976854920387268,
+      "mean_token_accuracy": 0.9968989025056362,
+      "num_tokens": 2892220.0,
+      "step": 3400
+    },
+    {
+      "entropy": 0.02088767145993188,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.032297272235155106,
+      "learning_rate": 1.0446841294298921e-05,
+      "loss": 0.011026575565338134,
+      "mean_token_accuracy": 0.9964779444038868,
+      "num_tokens": 3878184.0,
+      "step": 3500
+    },
+    {
+      "entropy": 0.018377634914308463,
+      "epoch": 0.384,
+      "grad_norm": 0.012006225995719433,
+      "learning_rate": 1.3467555555555556e-05,
+      "loss": 0.008617221661235975,
+      "mean_token_accuracy": 0.9970365500320559,
+      "num_tokens": 904843.0,
+      "step": 3600
+    },
+    {
+      "entropy": 0.016050403744447977,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 0.011128585785627365,
+      "learning_rate": 1.3378666666666666e-05,
+      "loss": 0.006250782608985901,
+      "mean_token_accuracy": 0.9976212471723557,
+      "num_tokens": 1878860.0,
+      "step": 3700
+    },
+    {
+      "entropy": 0.01809800002258271,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 0.01952100545167923,
+      "learning_rate": 1.3289777777777778e-05,
+      "loss": 0.008064679503440857,
+      "mean_token_accuracy": 0.9971583542227745,
+      "num_tokens": 2860744.0,
+      "step": 3800
+    },
+    {
+      "entropy": 0.019823117861524225,
+      "epoch": 0.416,
+      "grad_norm": 0.011735321022570133,
+      "learning_rate": 1.3200888888888889e-05,
+      "loss": 0.010386246442794799,
+      "mean_token_accuracy": 0.9966941741108895,
+      "num_tokens": 3831565.0,
+      "step": 3900
+    },
+    {
+      "entropy": 0.02049923066298889,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.006284466944634914,
+      "learning_rate": 1.3112e-05,
+      "loss": 0.01045264061107192,
+      "mean_token_accuracy": 0.9964593903616418,
+      "num_tokens": 849872.0,
+      "step": 4000
+    },
+    {
+      "entropy": 0.017120514765847476,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 0.010025433264672756,
+      "learning_rate": 1.3023111111111111e-05,
+      "loss": 0.007535084486007691,
+      "mean_token_accuracy": 0.9973040929436684,
+      "num_tokens": 1826365.0,
+      "step": 4100
+    },
+    {
+      "entropy": 0.01937343619065359,
+      "epoch": 0.448,
+      "grad_norm": 0.008356385864317417,
+      "learning_rate": 1.2934222222222222e-05,
+      "loss": 0.010190980434417725,
+      "mean_token_accuracy": 0.9967715987563133,
+      "num_tokens": 2802749.0,
+      "step": 4200
+    },
+    {
+      "entropy": 0.017232202125014737,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 0.011389357037842274,
+      "learning_rate": 1.2845333333333334e-05,
+      "loss": 0.007943087816238403,
+      "mean_token_accuracy": 0.9973043432831764,
+      "num_tokens": 3802758.0,
+      "step": 4300
+    },
+    {
+      "entropy": 0.022822092252748984,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 0.017312563955783844,
+      "learning_rate": 1.2756444444444444e-05,
+      "loss": 0.012838510819423346,
+      "mean_token_accuracy": 0.9959132893953795,
+      "num_tokens": 789736.0,
+      "step": 4400
+    },
+    {
+      "entropy": 0.0184966369275935,
+      "epoch": 0.48,
+      "grad_norm": 0.04721185564994812,
+      "learning_rate": 1.2667555555555557e-05,
+      "loss": 0.009318522214889526,
+      "mean_token_accuracy": 0.9968563948571681,
+      "num_tokens": 1767600.0,
+      "step": 4500
+    },
+    {
+      "entropy": 0.021548866296652706,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 0.9198243021965027,
+      "learning_rate": 1.2578666666666667e-05,
+      "loss": 0.013902791738510133,
+      "mean_token_accuracy": 0.9959381237626076,
+      "num_tokens": 2754982.0,
+      "step": 4600
+    },
+    {
+      "entropy": 0.0234364516264759,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 0.011652004905045033,
+      "learning_rate": 1.2489777777777779e-05,
+      "loss": 0.011085785627365112,
+      "mean_token_accuracy": 0.9966410009562969,
+      "num_tokens": 3730109.0,
+      "step": 4700
+    },
+    {
+      "entropy": 0.017517962178529856,
+      "epoch": 0.512,
+      "grad_norm": 0.008591280318796635,
+      "learning_rate": 1.240088888888889e-05,
+      "loss": 0.0068675024168831965,
+      "mean_token_accuracy": 0.9973819294533173,
+      "num_tokens": 753786.0,
+      "step": 4800
+    },
+    {
+      "entropy": 0.019476991441333667,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 0.013633953407406807,
+      "learning_rate": 1.2312e-05,
+      "loss": 0.009302983283996582,
+      "mean_token_accuracy": 0.9968732745945453,
+      "num_tokens": 1732738.0,
+      "step": 4900
+    },
+    {
+      "entropy": 0.020917424112558366,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.01434489618986845,
+      "learning_rate": 1.2223111111111112e-05,
+      "loss": 0.010165597200393678,
+      "mean_token_accuracy": 0.9965643344819546,
+      "num_tokens": 2702069.0,
+      "step": 5000
+    },
+    {
+      "entropy": 0.017627096675569193,
+      "epoch": 0.544,
+      "grad_norm": 0.010028124786913395,
+      "learning_rate": 1.2134222222222223e-05,
+      "loss": 0.008581981062889099,
+      "mean_token_accuracy": 0.9972231885790825,
+      "num_tokens": 3692514.0,
+      "step": 5100
+    },
+    {
+      "entropy": 0.018209505494293832,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 0.010386968962848186,
+      "learning_rate": 1.2045333333333333e-05,
+      "loss": 0.00814423689971099,
+      "mean_token_accuracy": 0.9972281313023051,
+      "num_tokens": 739184.0,
+      "step": 5200
+    },
+    {
+      "entropy": 0.02268622429575771,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 0.01215888187289238,
+      "learning_rate": 1.1956444444444445e-05,
+      "loss": 0.012614715099334716,
+      "mean_token_accuracy": 0.9960222035646439,
+      "num_tokens": 1710404.0,
+      "step": 5300
+    },
+    {
+      "entropy": 0.020103816259652376,
+      "epoch": 0.576,
+      "grad_norm": 0.014174265787005424,
+      "learning_rate": 1.1867555555555556e-05,
+      "loss": 0.010200117826461791,
+      "mean_token_accuracy": 0.9967270520329475,
+      "num_tokens": 2686777.0,
+      "step": 5400
+    },
+    {
+      "entropy": 0.020467385197989643,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 0.014257642440497875,
+      "learning_rate": 1.1778666666666666e-05,
+      "loss": 0.010185201168060303,
+      "mean_token_accuracy": 0.9966062535345555,
+      "num_tokens": 3655445.0,
+      "step": 5500
+    },
+    {
+      "entropy": 0.020591240752474878,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 0.011690130457282066,
+      "learning_rate": 1.1689777777777778e-05,
+      "loss": 0.010783259419427402,
+      "mean_token_accuracy": 0.9964483192433482,
+      "num_tokens": 686164.0,
+      "step": 5600
+    },
+    {
+      "entropy": 0.0205141630244907,
+      "epoch": 0.608,
+      "grad_norm": 0.006469405256211758,
+      "learning_rate": 1.160088888888889e-05,
+      "loss": 0.010492314100265503,
+      "mean_token_accuracy": 0.9966046234965324,
+      "num_tokens": 1662167.0,
+      "step": 5700
+    },
+    {
+      "entropy": 0.020147288321750237,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.01614871807396412,
+      "learning_rate": 1.1512e-05,
+      "loss": 0.010045292377471924,
+      "mean_token_accuracy": 0.9966985350847244,
+      "num_tokens": 2643221.0,
+      "step": 5800
+    },
+    {
+      "entropy": 0.01912423676229082,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 0.010019957087934017,
+      "learning_rate": 1.1423111111111111e-05,
+      "loss": 0.009680591821670533,
+      "mean_token_accuracy": 0.9968334528803825,
+      "num_tokens": 3611899.0,
+      "step": 5900
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.419013500213623,
+      "learning_rate": 1.2e-05,
+      "loss": 0.0443,
+      "step": 6000
+    },
+    {
+      "epoch": 0.16266666666666665,
+      "grad_norm": 0.0630747601389885,
+      "learning_rate": 1.22e-05,
+      "loss": 0.0066,
+      "step": 6100
+    },
+    {
+      "epoch": 0.16533333333333333,
+      "grad_norm": 0.08701734989881516,
+      "learning_rate": 1.24e-05,
+      "loss": 0.0171,
+      "step": 6200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 0.7274932861328125,
+      "learning_rate": 1.26e-05,
+      "loss": 0.0065,
+      "step": 6300
+    },
+    {
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.07388290017843246,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 0.0087,
+      "step": 6400
+    },
+    {
+      "epoch": 0.17333333333333334,
+      "grad_norm": 0.045733992010354996,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.0114,
+      "step": 6500
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 0.4374091625213623,
+      "learning_rate": 1.32e-05,
+      "loss": 0.0139,
+      "step": 6600
+    },
+    {
+      "epoch": 0.17866666666666667,
+      "grad_norm": 0.050165899097919464,
+      "learning_rate": 1.34e-05,
+      "loss": 0.0145,
+      "step": 6700
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 75000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.530244817910268e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-6700/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c40a4c02bff4c325a39b29bbad54a640a82b1bc361c29cc2656ac8d29cf43eaa
+size 5432

checkpoint-6800/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Dorn4449/CyberSentinel-Mistral-7B-v3.8
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

checkpoint-6800/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6800/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0507d10eae1f0346bd6339d599120583923b5ce79cf6cee34102ef3cee8c9d98
+size 27297032

checkpoint-6800/checkpoint-6000/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6800/checkpoint-6000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:286625e19ea92f6bcbc715bf5d40bac2ac85da8b9ecb31679d50c38cb4b4b694
+size 27297032

checkpoint-6800/checkpoint-6100/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6800/checkpoint-6100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a6ee2942c85113263a2331006fa30db088217df193b0fca3de2eebe282399b
+size 27297032

checkpoint-6800/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99af4a4481c6650369d5e07dbe438199eb0c51fb7785d01f896dc75efa81c1cb
+size 54744314

checkpoint-6800/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ce03ec18080cb529d92960e443bce2c42c82d7d8e70eff7f20951893977d067
+size 14244

checkpoint-6800/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82cccbba262cf26956ffd28c659fa3c037c0ffdf3b06154be11735905ca57f70
+size 1064

checkpoint-6800/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-6800/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

checkpoint-6800/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-6800/trainer_state.json

@@ -0,0 +1,686 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.18133333333333335,
+  "eval_steps": 500,
+  "global_step": 6800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.084044404476881,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 0.796875,
+      "learning_rate": 7.92e-07,
+      "loss": 1.3782466125488282,
+      "mean_token_accuracy": 0.7083857330679894,
+      "num_tokens": 986491.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1007767178118228,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 0.3125,
+      "learning_rate": 1.592e-06,
+      "loss": 1.2902149963378906,
+      "mean_token_accuracy": 0.7171274860203266,
+      "num_tokens": 1952567.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0802835547924041,
+      "epoch": 0.032,
+      "grad_norm": 0.28515625,
+      "learning_rate": 2.392e-06,
+      "loss": 1.170371322631836,
+      "mean_token_accuracy": 0.7354862231016159,
+      "num_tokens": 2935479.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0573877203464508,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 0.23046875,
+      "learning_rate": 3.192e-06,
+      "loss": 1.062843780517578,
+      "mean_token_accuracy": 0.7536254300177098,
+      "num_tokens": 3920344.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0149462178349495,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.212890625,
+      "learning_rate": 3.992e-06,
+      "loss": 0.9590372467041015,
+      "mean_token_accuracy": 0.7738636130094528,
+      "num_tokens": 4899217.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.8931056271493435,
+      "epoch": 0.064,
+      "grad_norm": 0.28125,
+      "learning_rate": 4.792e-06,
+      "loss": 0.7998863983154297,
+      "mean_token_accuracy": 0.8012332341074944,
+      "num_tokens": 5883391.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.6136953190714121,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 1.1015625,
+      "learning_rate": 5.592000000000001e-06,
+      "loss": 0.5131131362915039,
+      "mean_token_accuracy": 0.866335421949625,
+      "num_tokens": 6857125.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.2726459547691047,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.474609375,
+      "learning_rate": 6.392e-06,
+      "loss": 0.18439870834350586,
+      "mean_token_accuracy": 0.9579810079932213,
+      "num_tokens": 7831290.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.11284050881862641,
+      "epoch": 0.096,
+      "grad_norm": 0.1875,
+      "learning_rate": 7.192e-06,
+      "loss": 0.05271556854248047,
+      "mean_token_accuracy": 0.9904264670610428,
+      "num_tokens": 8815417.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.068436967888847,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.125,
+      "learning_rate": 7.992e-06,
+      "loss": 0.02549468755722046,
+      "mean_token_accuracy": 0.9951217715442181,
+      "num_tokens": 9782033.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.052372096767649055,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 8.792e-06,
+      "loss": 0.017259199619293213,
+      "mean_token_accuracy": 0.9962555834650993,
+      "num_tokens": 10759962.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.03862797610927373,
+      "epoch": 0.128,
+      "grad_norm": 0.0867946669459343,
+      "learning_rate": 9.591999999999999e-06,
+      "loss": 0.013324768543243408,
+      "mean_token_accuracy": 0.9968111206591129,
+      "num_tokens": 987986.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.032093781144358215,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 0.037592533975839615,
+      "learning_rate": 1.0392e-05,
+      "loss": 0.013392001390457153,
+      "mean_token_accuracy": 0.9965760576725006,
+      "num_tokens": 1968127.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.031904329673852774,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 0.10515860468149185,
+      "learning_rate": 1.1192e-05,
+      "loss": 0.016373103857040404,
+      "mean_token_accuracy": 0.9955054900050163,
+      "num_tokens": 2965677.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.0265167937008664,
+      "epoch": 0.16,
+      "grad_norm": 0.12689532339572906,
+      "learning_rate": 1.1992e-05,
+      "loss": 0.011554093360900878,
+      "mean_token_accuracy": 0.996508517563343,
+      "num_tokens": 3938144.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.024146563813555986,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.03401608020067215,
+      "learning_rate": 1.2792e-05,
+      "loss": 0.010398292541503906,
+      "mean_token_accuracy": 0.9966818282008171,
+      "num_tokens": 4897019.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.025755486716516316,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.05834396556019783,
+      "learning_rate": 1.3592000000000001e-05,
+      "loss": 0.012847075462341309,
+      "mean_token_accuracy": 0.9960671140253544,
+      "num_tokens": 5880125.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.023504739217460154,
+      "epoch": 0.192,
+      "grad_norm": 0.09117468446493149,
+      "learning_rate": 1.4392e-05,
+      "loss": 0.011869451999664306,
+      "mean_token_accuracy": 0.9964955732226372,
+      "num_tokens": 6867360.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.022944011739455164,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 0.046192608773708344,
+      "learning_rate": 1.4978666666666668e-05,
+      "loss": 0.010620262622833252,
+      "mean_token_accuracy": 0.9964672869443894,
+      "num_tokens": 7840472.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.018894561287015676,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.02761668898165226,
+      "learning_rate": 1.4889777777777778e-05,
+      "loss": 0.0067046540975570675,
+      "mean_token_accuracy": 0.9975774252414703,
+      "num_tokens": 8827474.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.020934174589347095,
+      "epoch": 0.224,
+      "grad_norm": 0.02931295707821846,
+      "learning_rate": 1.4800888888888889e-05,
+      "loss": 0.010140993595123292,
+      "mean_token_accuracy": 0.996714953482151,
+      "num_tokens": 9808475.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.020719884738791734,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 0.02652113325893879,
+      "learning_rate": 1.4712e-05,
+      "loss": 0.010557392835617066,
+      "mean_token_accuracy": 0.9969246552884579,
+      "num_tokens": 10782725.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.020559412932489068,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 0.032906673848629,
+      "learning_rate": 1.4623111111111113e-05,
+      "loss": 0.009789772033691406,
+      "mean_token_accuracy": 0.9966975942254066,
+      "num_tokens": 11769893.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.018388252432923764,
+      "epoch": 0.256,
+      "grad_norm": 0.08162333816289902,
+      "learning_rate": 1.4534222222222222e-05,
+      "loss": 0.007909480929374695,
+      "mean_token_accuracy": 0.9972486282885075,
+      "num_tokens": 12753597.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.02204789153067395,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.023511990904808044,
+      "learning_rate": 1.4445333333333334e-05,
+      "loss": 0.011945382356643677,
+      "mean_token_accuracy": 0.9963182592391968,
+      "num_tokens": 13739044.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.02037038065260276,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 0.02014540508389473,
+      "learning_rate": 1.4356444444444446e-05,
+      "loss": 0.010165022611618042,
+      "mean_token_accuracy": 0.9967658732831478,
+      "num_tokens": 14709501.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.01947357293218374,
+      "epoch": 0.288,
+      "grad_norm": 0.01829116977751255,
+      "learning_rate": 1.4267555555555555e-05,
+      "loss": 0.009100326895713806,
+      "mean_token_accuracy": 0.9969173397123814,
+      "num_tokens": 15693309.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.021565243480727078,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 0.031732361763715744,
+      "learning_rate": 1.4178666666666667e-05,
+      "loss": 0.01107092022895813,
+      "mean_token_accuracy": 0.9963353677093982,
+      "num_tokens": 985520.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.02136099517461844,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 0.03870174661278725,
+      "learning_rate": 1.4089777777777779e-05,
+      "loss": 0.011569523811340332,
+      "mean_token_accuracy": 0.9964252272248268,
+      "num_tokens": 1969572.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.019074252294376492,
+      "epoch": 0.32,
+      "grad_norm": 0.01954864338040352,
+      "learning_rate": 1.4000888888888888e-05,
+      "loss": 0.008727578520774841,
+      "mean_token_accuracy": 0.9970971086621284,
+      "num_tokens": 2945404.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.018130726229865102,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 0.01979902759194374,
+      "learning_rate": 1.3912e-05,
+      "loss": 0.007898266315460206,
+      "mean_token_accuracy": 0.9971335357427598,
+      "num_tokens": 3925122.0,
+      "step": 3100
+    },
+    {
+      "entropy": 0.019400757937546587,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 0.030273959040641785,
+      "learning_rate": 1.0980206234443522e-05,
+      "loss": 0.009979066749413809,
+      "mean_token_accuracy": 0.9968534293584526,
+      "num_tokens": 945781.0,
+      "step": 3200
+    },
+    {
+      "entropy": 0.021496493350714446,
+      "epoch": 0.352,
+      "grad_norm": 0.023218955844640732,
+      "learning_rate": 1.0802417921061989e-05,
+      "loss": 0.011733214855194091,
+      "mean_token_accuracy": 0.996385814100504,
+      "num_tokens": 1919451.0,
+      "step": 3300
+    },
+    {
+      "entropy": 0.01932728004641831,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 0.014179096557199955,
+      "learning_rate": 1.0624629607680455e-05,
+      "loss": 0.00976854920387268,
+      "mean_token_accuracy": 0.9968989025056362,
+      "num_tokens": 2892220.0,
+      "step": 3400
+    },
+    {
+      "entropy": 0.02088767145993188,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.032297272235155106,
+      "learning_rate": 1.0446841294298921e-05,
+      "loss": 0.011026575565338134,
+      "mean_token_accuracy": 0.9964779444038868,
+      "num_tokens": 3878184.0,
+      "step": 3500
+    },
+    {
+      "entropy": 0.018377634914308463,
+      "epoch": 0.384,
+      "grad_norm": 0.012006225995719433,
+      "learning_rate": 1.3467555555555556e-05,
+      "loss": 0.008617221661235975,
+      "mean_token_accuracy": 0.9970365500320559,
+      "num_tokens": 904843.0,
+      "step": 3600
+    },
+    {
+      "entropy": 0.016050403744447977,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 0.011128585785627365,
+      "learning_rate": 1.3378666666666666e-05,
+      "loss": 0.006250782608985901,
+      "mean_token_accuracy": 0.9976212471723557,
+      "num_tokens": 1878860.0,
+      "step": 3700
+    },
+    {
+      "entropy": 0.01809800002258271,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 0.01952100545167923,
+      "learning_rate": 1.3289777777777778e-05,
+      "loss": 0.008064679503440857,
+      "mean_token_accuracy": 0.9971583542227745,
+      "num_tokens": 2860744.0,
+      "step": 3800
+    },
+    {
+      "entropy": 0.019823117861524225,
+      "epoch": 0.416,
+      "grad_norm": 0.011735321022570133,
+      "learning_rate": 1.3200888888888889e-05,
+      "loss": 0.010386246442794799,
+      "mean_token_accuracy": 0.9966941741108895,
+      "num_tokens": 3831565.0,
+      "step": 3900
+    },
+    {
+      "entropy": 0.02049923066298889,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.006284466944634914,
+      "learning_rate": 1.3112e-05,
+      "loss": 0.01045264061107192,
+      "mean_token_accuracy": 0.9964593903616418,
+      "num_tokens": 849872.0,
+      "step": 4000
+    },
+    {
+      "entropy": 0.017120514765847476,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 0.010025433264672756,
+      "learning_rate": 1.3023111111111111e-05,
+      "loss": 0.007535084486007691,
+      "mean_token_accuracy": 0.9973040929436684,
+      "num_tokens": 1826365.0,
+      "step": 4100
+    },
+    {
+      "entropy": 0.01937343619065359,
+      "epoch": 0.448,
+      "grad_norm": 0.008356385864317417,
+      "learning_rate": 1.2934222222222222e-05,
+      "loss": 0.010190980434417725,
+      "mean_token_accuracy": 0.9967715987563133,
+      "num_tokens": 2802749.0,
+      "step": 4200
+    },
+    {
+      "entropy": 0.017232202125014737,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 0.011389357037842274,
+      "learning_rate": 1.2845333333333334e-05,
+      "loss": 0.007943087816238403,
+      "mean_token_accuracy": 0.9973043432831764,
+      "num_tokens": 3802758.0,
+      "step": 4300
+    },
+    {
+      "entropy": 0.022822092252748984,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 0.017312563955783844,
+      "learning_rate": 1.2756444444444444e-05,
+      "loss": 0.012838510819423346,
+      "mean_token_accuracy": 0.9959132893953795,
+      "num_tokens": 789736.0,
+      "step": 4400
+    },
+    {
+      "entropy": 0.0184966369275935,
+      "epoch": 0.48,
+      "grad_norm": 0.04721185564994812,
+      "learning_rate": 1.2667555555555557e-05,
+      "loss": 0.009318522214889526,
+      "mean_token_accuracy": 0.9968563948571681,
+      "num_tokens": 1767600.0,
+      "step": 4500
+    },
+    {
+      "entropy": 0.021548866296652706,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 0.9198243021965027,
+      "learning_rate": 1.2578666666666667e-05,
+      "loss": 0.013902791738510133,
+      "mean_token_accuracy": 0.9959381237626076,
+      "num_tokens": 2754982.0,
+      "step": 4600
+    },
+    {
+      "entropy": 0.0234364516264759,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 0.011652004905045033,
+      "learning_rate": 1.2489777777777779e-05,
+      "loss": 0.011085785627365112,
+      "mean_token_accuracy": 0.9966410009562969,
+      "num_tokens": 3730109.0,
+      "step": 4700
+    },
+    {
+      "entropy": 0.017517962178529856,
+      "epoch": 0.512,
+      "grad_norm": 0.008591280318796635,
+      "learning_rate": 1.240088888888889e-05,
+      "loss": 0.0068675024168831965,
+      "mean_token_accuracy": 0.9973819294533173,
+      "num_tokens": 753786.0,
+      "step": 4800
+    },
+    {
+      "entropy": 0.019476991441333667,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 0.013633953407406807,
+      "learning_rate": 1.2312e-05,
+      "loss": 0.009302983283996582,
+      "mean_token_accuracy": 0.9968732745945453,
+      "num_tokens": 1732738.0,
+      "step": 4900
+    },
+    {
+      "entropy": 0.020917424112558366,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.01434489618986845,
+      "learning_rate": 1.2223111111111112e-05,
+      "loss": 0.010165597200393678,
+      "mean_token_accuracy": 0.9965643344819546,
+      "num_tokens": 2702069.0,
+      "step": 5000
+    },
+    {
+      "entropy": 0.017627096675569193,
+      "epoch": 0.544,
+      "grad_norm": 0.010028124786913395,
+      "learning_rate": 1.2134222222222223e-05,
+      "loss": 0.008581981062889099,
+      "mean_token_accuracy": 0.9972231885790825,
+      "num_tokens": 3692514.0,
+      "step": 5100
+    },
+    {
+      "entropy": 0.018209505494293832,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 0.010386968962848186,
+      "learning_rate": 1.2045333333333333e-05,
+      "loss": 0.00814423689971099,
+      "mean_token_accuracy": 0.9972281313023051,
+      "num_tokens": 739184.0,
+      "step": 5200
+    },
+    {
+      "entropy": 0.02268622429575771,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 0.01215888187289238,
+      "learning_rate": 1.1956444444444445e-05,
+      "loss": 0.012614715099334716,
+      "mean_token_accuracy": 0.9960222035646439,
+      "num_tokens": 1710404.0,
+      "step": 5300
+    },
+    {
+      "entropy": 0.020103816259652376,
+      "epoch": 0.576,
+      "grad_norm": 0.014174265787005424,
+      "learning_rate": 1.1867555555555556e-05,
+      "loss": 0.010200117826461791,
+      "mean_token_accuracy": 0.9967270520329475,
+      "num_tokens": 2686777.0,
+      "step": 5400
+    },
+    {
+      "entropy": 0.020467385197989643,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 0.014257642440497875,
+      "learning_rate": 1.1778666666666666e-05,
+      "loss": 0.010185201168060303,
+      "mean_token_accuracy": 0.9966062535345555,
+      "num_tokens": 3655445.0,
+      "step": 5500
+    },
+    {
+      "entropy": 0.020591240752474878,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 0.011690130457282066,
+      "learning_rate": 1.1689777777777778e-05,
+      "loss": 0.010783259419427402,
+      "mean_token_accuracy": 0.9964483192433482,
+      "num_tokens": 686164.0,
+      "step": 5600
+    },
+    {
+      "entropy": 0.0205141630244907,
+      "epoch": 0.608,
+      "grad_norm": 0.006469405256211758,
+      "learning_rate": 1.160088888888889e-05,
+      "loss": 0.010492314100265503,
+      "mean_token_accuracy": 0.9966046234965324,
+      "num_tokens": 1662167.0,
+      "step": 5700
+    },
+    {
+      "entropy": 0.020147288321750237,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.01614871807396412,
+      "learning_rate": 1.1512e-05,
+      "loss": 0.010045292377471924,
+      "mean_token_accuracy": 0.9966985350847244,
+      "num_tokens": 2643221.0,
+      "step": 5800
+    },
+    {
+      "entropy": 0.01912423676229082,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 0.010019957087934017,
+      "learning_rate": 1.1423111111111111e-05,
+      "loss": 0.009680591821670533,
+      "mean_token_accuracy": 0.9968334528803825,
+      "num_tokens": 3611899.0,
+      "step": 5900
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.419013500213623,
+      "learning_rate": 1.2e-05,
+      "loss": 0.0443,
+      "step": 6000
+    },
+    {
+      "epoch": 0.16266666666666665,
+      "grad_norm": 0.0630747601389885,
+      "learning_rate": 1.22e-05,
+      "loss": 0.0066,
+      "step": 6100
+    },
+    {
+      "epoch": 0.16533333333333333,
+      "grad_norm": 0.08701734989881516,
+      "learning_rate": 1.24e-05,
+      "loss": 0.0171,
+      "step": 6200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 0.7274932861328125,
+      "learning_rate": 1.26e-05,
+      "loss": 0.0065,
+      "step": 6300
+    },
+    {
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.07388290017843246,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 0.0087,
+      "step": 6400
+    },
+    {
+      "epoch": 0.17333333333333334,
+      "grad_norm": 0.045733992010354996,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.0114,
+      "step": 6500
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 0.4374091625213623,
+      "learning_rate": 1.32e-05,
+      "loss": 0.0139,
+      "step": 6600
+    },
+    {
+      "epoch": 0.17866666666666667,
+      "grad_norm": 0.050165899097919464,
+      "learning_rate": 1.34e-05,
+      "loss": 0.0145,
+      "step": 6700
+    },
+    {
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.047875139862298965,
+      "learning_rate": 1.36e-05,
+      "loss": 0.0101,
+      "step": 6800
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 75000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.5446173650588467e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-6800/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c40a4c02bff4c325a39b29bbad54a640a82b1bc361c29cc2656ac8d29cf43eaa
+size 5432

checkpoint-6900/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Dorn4449/CyberSentinel-Mistral-7B-v3.8
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

checkpoint-6900/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6900/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fb16bd15c2d30833ede00f37a172efbf1cbd478e9aa01d936051c4064a692aa
+size 27297032

checkpoint-6900/checkpoint-6000/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6900/checkpoint-6000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:286625e19ea92f6bcbc715bf5d40bac2ac85da8b9ecb31679d50c38cb4b4b694
+size 27297032

checkpoint-6900/checkpoint-6100/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Dorn4449/CyberSentinel-Mistral-7B-v3.8",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "v_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-6900/checkpoint-6100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a6ee2942c85113263a2331006fa30db088217df193b0fca3de2eebe282399b
+size 27297032

checkpoint-6900/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58edf8e29a66243e02f52a2373020e2d5895e90828764e844d326a9e3ca53dbd
+size 54744314

checkpoint-6900/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6bd7d612ceb3e709a62706076d7ac3f7c5e669a74c0255776051d19f8d03b0c
+size 14244

checkpoint-6900/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a98d5fe154c6c875e5ee23462b9940fd18916f76deaf1068c478a58799aa59cf
+size 1064

checkpoint-6900/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-6900/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

checkpoint-6900/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + eos_token}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-6900/trainer_state.json

@@ -0,0 +1,693 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.184,
+  "eval_steps": 500,
+  "global_step": 6900,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.084044404476881,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 0.796875,
+      "learning_rate": 7.92e-07,
+      "loss": 1.3782466125488282,
+      "mean_token_accuracy": 0.7083857330679894,
+      "num_tokens": 986491.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.1007767178118228,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 0.3125,
+      "learning_rate": 1.592e-06,
+      "loss": 1.2902149963378906,
+      "mean_token_accuracy": 0.7171274860203266,
+      "num_tokens": 1952567.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0802835547924041,
+      "epoch": 0.032,
+      "grad_norm": 0.28515625,
+      "learning_rate": 2.392e-06,
+      "loss": 1.170371322631836,
+      "mean_token_accuracy": 0.7354862231016159,
+      "num_tokens": 2935479.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0573877203464508,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 0.23046875,
+      "learning_rate": 3.192e-06,
+      "loss": 1.062843780517578,
+      "mean_token_accuracy": 0.7536254300177098,
+      "num_tokens": 3920344.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0149462178349495,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.212890625,
+      "learning_rate": 3.992e-06,
+      "loss": 0.9590372467041015,
+      "mean_token_accuracy": 0.7738636130094528,
+      "num_tokens": 4899217.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.8931056271493435,
+      "epoch": 0.064,
+      "grad_norm": 0.28125,
+      "learning_rate": 4.792e-06,
+      "loss": 0.7998863983154297,
+      "mean_token_accuracy": 0.8012332341074944,
+      "num_tokens": 5883391.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.6136953190714121,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 1.1015625,
+      "learning_rate": 5.592000000000001e-06,
+      "loss": 0.5131131362915039,
+      "mean_token_accuracy": 0.866335421949625,
+      "num_tokens": 6857125.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.2726459547691047,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.474609375,
+      "learning_rate": 6.392e-06,
+      "loss": 0.18439870834350586,
+      "mean_token_accuracy": 0.9579810079932213,
+      "num_tokens": 7831290.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.11284050881862641,
+      "epoch": 0.096,
+      "grad_norm": 0.1875,
+      "learning_rate": 7.192e-06,
+      "loss": 0.05271556854248047,
+      "mean_token_accuracy": 0.9904264670610428,
+      "num_tokens": 8815417.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.068436967888847,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.125,
+      "learning_rate": 7.992e-06,
+      "loss": 0.02549468755722046,
+      "mean_token_accuracy": 0.9951217715442181,
+      "num_tokens": 9782033.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.052372096767649055,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 8.792e-06,
+      "loss": 0.017259199619293213,
+      "mean_token_accuracy": 0.9962555834650993,
+      "num_tokens": 10759962.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.03862797610927373,
+      "epoch": 0.128,
+      "grad_norm": 0.0867946669459343,
+      "learning_rate": 9.591999999999999e-06,
+      "loss": 0.013324768543243408,
+      "mean_token_accuracy": 0.9968111206591129,
+      "num_tokens": 987986.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.032093781144358215,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 0.037592533975839615,
+      "learning_rate": 1.0392e-05,
+      "loss": 0.013392001390457153,
+      "mean_token_accuracy": 0.9965760576725006,
+      "num_tokens": 1968127.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.031904329673852774,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 0.10515860468149185,
+      "learning_rate": 1.1192e-05,
+      "loss": 0.016373103857040404,
+      "mean_token_accuracy": 0.9955054900050163,
+      "num_tokens": 2965677.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.0265167937008664,
+      "epoch": 0.16,
+      "grad_norm": 0.12689532339572906,
+      "learning_rate": 1.1992e-05,
+      "loss": 0.011554093360900878,
+      "mean_token_accuracy": 0.996508517563343,
+      "num_tokens": 3938144.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.024146563813555986,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.03401608020067215,
+      "learning_rate": 1.2792e-05,
+      "loss": 0.010398292541503906,
+      "mean_token_accuracy": 0.9966818282008171,
+      "num_tokens": 4897019.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.025755486716516316,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.05834396556019783,
+      "learning_rate": 1.3592000000000001e-05,
+      "loss": 0.012847075462341309,
+      "mean_token_accuracy": 0.9960671140253544,
+      "num_tokens": 5880125.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.023504739217460154,
+      "epoch": 0.192,
+      "grad_norm": 0.09117468446493149,
+      "learning_rate": 1.4392e-05,
+      "loss": 0.011869451999664306,
+      "mean_token_accuracy": 0.9964955732226372,
+      "num_tokens": 6867360.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.022944011739455164,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 0.046192608773708344,
+      "learning_rate": 1.4978666666666668e-05,
+      "loss": 0.010620262622833252,
+      "mean_token_accuracy": 0.9964672869443894,
+      "num_tokens": 7840472.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.018894561287015676,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.02761668898165226,
+      "learning_rate": 1.4889777777777778e-05,
+      "loss": 0.0067046540975570675,
+      "mean_token_accuracy": 0.9975774252414703,
+      "num_tokens": 8827474.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.020934174589347095,
+      "epoch": 0.224,
+      "grad_norm": 0.02931295707821846,
+      "learning_rate": 1.4800888888888889e-05,
+      "loss": 0.010140993595123292,
+      "mean_token_accuracy": 0.996714953482151,
+      "num_tokens": 9808475.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.020719884738791734,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 0.02652113325893879,
+      "learning_rate": 1.4712e-05,
+      "loss": 0.010557392835617066,
+      "mean_token_accuracy": 0.9969246552884579,
+      "num_tokens": 10782725.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.020559412932489068,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 0.032906673848629,
+      "learning_rate": 1.4623111111111113e-05,
+      "loss": 0.009789772033691406,
+      "mean_token_accuracy": 0.9966975942254066,
+      "num_tokens": 11769893.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.018388252432923764,
+      "epoch": 0.256,
+      "grad_norm": 0.08162333816289902,
+      "learning_rate": 1.4534222222222222e-05,
+      "loss": 0.007909480929374695,
+      "mean_token_accuracy": 0.9972486282885075,
+      "num_tokens": 12753597.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.02204789153067395,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.023511990904808044,
+      "learning_rate": 1.4445333333333334e-05,
+      "loss": 0.011945382356643677,
+      "mean_token_accuracy": 0.9963182592391968,
+      "num_tokens": 13739044.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.02037038065260276,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 0.02014540508389473,
+      "learning_rate": 1.4356444444444446e-05,
+      "loss": 0.010165022611618042,
+      "mean_token_accuracy": 0.9967658732831478,
+      "num_tokens": 14709501.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.01947357293218374,
+      "epoch": 0.288,
+      "grad_norm": 0.01829116977751255,
+      "learning_rate": 1.4267555555555555e-05,
+      "loss": 0.009100326895713806,
+      "mean_token_accuracy": 0.9969173397123814,
+      "num_tokens": 15693309.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.021565243480727078,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 0.031732361763715744,
+      "learning_rate": 1.4178666666666667e-05,
+      "loss": 0.01107092022895813,
+      "mean_token_accuracy": 0.9963353677093982,
+      "num_tokens": 985520.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.02136099517461844,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 0.03870174661278725,
+      "learning_rate": 1.4089777777777779e-05,
+      "loss": 0.011569523811340332,
+      "mean_token_accuracy": 0.9964252272248268,
+      "num_tokens": 1969572.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.019074252294376492,
+      "epoch": 0.32,
+      "grad_norm": 0.01954864338040352,
+      "learning_rate": 1.4000888888888888e-05,
+      "loss": 0.008727578520774841,
+      "mean_token_accuracy": 0.9970971086621284,
+      "num_tokens": 2945404.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.018130726229865102,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 0.01979902759194374,
+      "learning_rate": 1.3912e-05,
+      "loss": 0.007898266315460206,
+      "mean_token_accuracy": 0.9971335357427598,
+      "num_tokens": 3925122.0,
+      "step": 3100
+    },
+    {
+      "entropy": 0.019400757937546587,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 0.030273959040641785,
+      "learning_rate": 1.0980206234443522e-05,
+      "loss": 0.009979066749413809,
+      "mean_token_accuracy": 0.9968534293584526,
+      "num_tokens": 945781.0,
+      "step": 3200
+    },
+    {
+      "entropy": 0.021496493350714446,
+      "epoch": 0.352,
+      "grad_norm": 0.023218955844640732,
+      "learning_rate": 1.0802417921061989e-05,
+      "loss": 0.011733214855194091,
+      "mean_token_accuracy": 0.996385814100504,
+      "num_tokens": 1919451.0,
+      "step": 3300
+    },
+    {
+      "entropy": 0.01932728004641831,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 0.014179096557199955,
+      "learning_rate": 1.0624629607680455e-05,
+      "loss": 0.00976854920387268,
+      "mean_token_accuracy": 0.9968989025056362,
+      "num_tokens": 2892220.0,
+      "step": 3400
+    },
+    {
+      "entropy": 0.02088767145993188,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.032297272235155106,
+      "learning_rate": 1.0446841294298921e-05,
+      "loss": 0.011026575565338134,
+      "mean_token_accuracy": 0.9964779444038868,
+      "num_tokens": 3878184.0,
+      "step": 3500
+    },
+    {
+      "entropy": 0.018377634914308463,
+      "epoch": 0.384,
+      "grad_norm": 0.012006225995719433,
+      "learning_rate": 1.3467555555555556e-05,
+      "loss": 0.008617221661235975,
+      "mean_token_accuracy": 0.9970365500320559,
+      "num_tokens": 904843.0,
+      "step": 3600
+    },
+    {
+      "entropy": 0.016050403744447977,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 0.011128585785627365,
+      "learning_rate": 1.3378666666666666e-05,
+      "loss": 0.006250782608985901,
+      "mean_token_accuracy": 0.9976212471723557,
+      "num_tokens": 1878860.0,
+      "step": 3700
+    },
+    {
+      "entropy": 0.01809800002258271,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 0.01952100545167923,
+      "learning_rate": 1.3289777777777778e-05,
+      "loss": 0.008064679503440857,
+      "mean_token_accuracy": 0.9971583542227745,
+      "num_tokens": 2860744.0,
+      "step": 3800
+    },
+    {
+      "entropy": 0.019823117861524225,
+      "epoch": 0.416,
+      "grad_norm": 0.011735321022570133,
+      "learning_rate": 1.3200888888888889e-05,
+      "loss": 0.010386246442794799,
+      "mean_token_accuracy": 0.9966941741108895,
+      "num_tokens": 3831565.0,
+      "step": 3900
+    },
+    {
+      "entropy": 0.02049923066298889,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.006284466944634914,
+      "learning_rate": 1.3112e-05,
+      "loss": 0.01045264061107192,
+      "mean_token_accuracy": 0.9964593903616418,
+      "num_tokens": 849872.0,
+      "step": 4000
+    },
+    {
+      "entropy": 0.017120514765847476,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 0.010025433264672756,
+      "learning_rate": 1.3023111111111111e-05,
+      "loss": 0.007535084486007691,
+      "mean_token_accuracy": 0.9973040929436684,
+      "num_tokens": 1826365.0,
+      "step": 4100
+    },
+    {
+      "entropy": 0.01937343619065359,
+      "epoch": 0.448,
+      "grad_norm": 0.008356385864317417,
+      "learning_rate": 1.2934222222222222e-05,
+      "loss": 0.010190980434417725,
+      "mean_token_accuracy": 0.9967715987563133,
+      "num_tokens": 2802749.0,
+      "step": 4200
+    },
+    {
+      "entropy": 0.017232202125014737,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 0.011389357037842274,
+      "learning_rate": 1.2845333333333334e-05,
+      "loss": 0.007943087816238403,
+      "mean_token_accuracy": 0.9973043432831764,
+      "num_tokens": 3802758.0,
+      "step": 4300
+    },
+    {
+      "entropy": 0.022822092252748984,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 0.017312563955783844,
+      "learning_rate": 1.2756444444444444e-05,
+      "loss": 0.012838510819423346,
+      "mean_token_accuracy": 0.9959132893953795,
+      "num_tokens": 789736.0,
+      "step": 4400
+    },
+    {
+      "entropy": 0.0184966369275935,
+      "epoch": 0.48,
+      "grad_norm": 0.04721185564994812,
+      "learning_rate": 1.2667555555555557e-05,
+      "loss": 0.009318522214889526,
+      "mean_token_accuracy": 0.9968563948571681,
+      "num_tokens": 1767600.0,
+      "step": 4500
+    },
+    {
+      "entropy": 0.021548866296652706,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 0.9198243021965027,
+      "learning_rate": 1.2578666666666667e-05,
+      "loss": 0.013902791738510133,
+      "mean_token_accuracy": 0.9959381237626076,
+      "num_tokens": 2754982.0,
+      "step": 4600
+    },
+    {
+      "entropy": 0.0234364516264759,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 0.011652004905045033,
+      "learning_rate": 1.2489777777777779e-05,
+      "loss": 0.011085785627365112,
+      "mean_token_accuracy": 0.9966410009562969,
+      "num_tokens": 3730109.0,
+      "step": 4700
+    },
+    {
+      "entropy": 0.017517962178529856,
+      "epoch": 0.512,
+      "grad_norm": 0.008591280318796635,
+      "learning_rate": 1.240088888888889e-05,
+      "loss": 0.0068675024168831965,
+      "mean_token_accuracy": 0.9973819294533173,
+      "num_tokens": 753786.0,
+      "step": 4800
+    },
+    {
+      "entropy": 0.019476991441333667,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 0.013633953407406807,
+      "learning_rate": 1.2312e-05,
+      "loss": 0.009302983283996582,
+      "mean_token_accuracy": 0.9968732745945453,
+      "num_tokens": 1732738.0,
+      "step": 4900
+    },
+    {
+      "entropy": 0.020917424112558366,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.01434489618986845,
+      "learning_rate": 1.2223111111111112e-05,
+      "loss": 0.010165597200393678,
+      "mean_token_accuracy": 0.9965643344819546,
+      "num_tokens": 2702069.0,
+      "step": 5000
+    },
+    {
+      "entropy": 0.017627096675569193,
+      "epoch": 0.544,
+      "grad_norm": 0.010028124786913395,
+      "learning_rate": 1.2134222222222223e-05,
+      "loss": 0.008581981062889099,
+      "mean_token_accuracy": 0.9972231885790825,
+      "num_tokens": 3692514.0,
+      "step": 5100
+    },
+    {
+      "entropy": 0.018209505494293832,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 0.010386968962848186,
+      "learning_rate": 1.2045333333333333e-05,
+      "loss": 0.00814423689971099,
+      "mean_token_accuracy": 0.9972281313023051,
+      "num_tokens": 739184.0,
+      "step": 5200
+    },
+    {
+      "entropy": 0.02268622429575771,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 0.01215888187289238,
+      "learning_rate": 1.1956444444444445e-05,
+      "loss": 0.012614715099334716,
+      "mean_token_accuracy": 0.9960222035646439,
+      "num_tokens": 1710404.0,
+      "step": 5300
+    },
+    {
+      "entropy": 0.020103816259652376,
+      "epoch": 0.576,
+      "grad_norm": 0.014174265787005424,
+      "learning_rate": 1.1867555555555556e-05,
+      "loss": 0.010200117826461791,
+      "mean_token_accuracy": 0.9967270520329475,
+      "num_tokens": 2686777.0,
+      "step": 5400
+    },
+    {
+      "entropy": 0.020467385197989643,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 0.014257642440497875,
+      "learning_rate": 1.1778666666666666e-05,
+      "loss": 0.010185201168060303,
+      "mean_token_accuracy": 0.9966062535345555,
+      "num_tokens": 3655445.0,
+      "step": 5500
+    },
+    {
+      "entropy": 0.020591240752474878,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 0.011690130457282066,
+      "learning_rate": 1.1689777777777778e-05,
+      "loss": 0.010783259419427402,
+      "mean_token_accuracy": 0.9964483192433482,
+      "num_tokens": 686164.0,
+      "step": 5600
+    },
+    {
+      "entropy": 0.0205141630244907,
+      "epoch": 0.608,
+      "grad_norm": 0.006469405256211758,
+      "learning_rate": 1.160088888888889e-05,
+      "loss": 0.010492314100265503,
+      "mean_token_accuracy": 0.9966046234965324,
+      "num_tokens": 1662167.0,
+      "step": 5700
+    },
+    {
+      "entropy": 0.020147288321750237,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.01614871807396412,
+      "learning_rate": 1.1512e-05,
+      "loss": 0.010045292377471924,
+      "mean_token_accuracy": 0.9966985350847244,
+      "num_tokens": 2643221.0,
+      "step": 5800
+    },
+    {
+      "entropy": 0.01912423676229082,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 0.010019957087934017,
+      "learning_rate": 1.1423111111111111e-05,
+      "loss": 0.009680591821670533,
+      "mean_token_accuracy": 0.9968334528803825,
+      "num_tokens": 3611899.0,
+      "step": 5900
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.419013500213623,
+      "learning_rate": 1.2e-05,
+      "loss": 0.0443,
+      "step": 6000
+    },
+    {
+      "epoch": 0.16266666666666665,
+      "grad_norm": 0.0630747601389885,
+      "learning_rate": 1.22e-05,
+      "loss": 0.0066,
+      "step": 6100
+    },
+    {
+      "epoch": 0.16533333333333333,
+      "grad_norm": 0.08701734989881516,
+      "learning_rate": 1.24e-05,
+      "loss": 0.0171,
+      "step": 6200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 0.7274932861328125,
+      "learning_rate": 1.26e-05,
+      "loss": 0.0065,
+      "step": 6300
+    },
+    {
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.07388290017843246,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 0.0087,
+      "step": 6400
+    },
+    {
+      "epoch": 0.17333333333333334,
+      "grad_norm": 0.045733992010354996,
+      "learning_rate": 1.3000000000000001e-05,
+      "loss": 0.0114,
+      "step": 6500
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 0.4374091625213623,
+      "learning_rate": 1.32e-05,
+      "loss": 0.0139,
+      "step": 6600
+    },
+    {
+      "epoch": 0.17866666666666667,
+      "grad_norm": 0.050165899097919464,
+      "learning_rate": 1.34e-05,
+      "loss": 0.0145,
+      "step": 6700
+    },
+    {
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.047875139862298965,
+      "learning_rate": 1.36e-05,
+      "loss": 0.0101,
+      "step": 6800
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 0.06959123909473419,
+      "learning_rate": 1.3800000000000002e-05,
+      "loss": 0.0126,
+      "step": 6900
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 75000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.559366092381192e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-6900/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c40a4c02bff4c325a39b29bbad54a640a82b1bc361c29cc2656ac8d29cf43eaa
+size 5432