Model save

README.md

@@ -0,0 +1,59 @@
+---
+base_model: hllj/mistral-vi-math
+tags:
+- generated_from_trainer
+model-index:
+- name: sft-mistral-v1-original-data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# sft-mistral-v1-original-data
+
+This model is a fine-tuned version of [hllj/mistral-vi-math](https://huggingface.co/hllj/mistral-vi-math) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5087
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 2
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.3103        | 1.27  | 200  | 0.5224          |
+
+
+### Framework versions
+
+- Transformers 4.35.2
+- Pytorch 2.1.0
+- Datasets 2.15.0
+- Tokenizers 0.15.0

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9f098cde9492e4e08a9e4630d9b6655df2c1a4d8159595ebc4007e34f209afb
+oid sha256:28e27032630142d5d3a5633ce86247205605fe887fce465f29e6f88d604023e2
 size 872450448

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 1.39,
+    "eval_loss": 0.508748471736908,
+    "eval_runtime": 6.855,
+    "eval_samples": 140,
+    "eval_samples_per_second": 20.423,
+    "eval_steps_per_second": 5.106,
+    "train_loss": 0.40742398091291976,
+    "train_runtime": 500.0224,
+    "train_samples": 1196,
+    "train_samples_per_second": 4.784,
+    "train_steps_per_second": 1.196
+}

config_argument.yaml

@@ -0,0 +1,52 @@
+cache_dir: ./cache
+ddp_find_unused_parameters: false
+ddp_timeout: 30000
+device_map: auto
+do_eval: true
+do_train: true
+eval_steps: 200
+evaluation_strategy: steps
+fp16: true
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: false
+hub_model_id: hllj/sft-mistral-v1-original-data
+hub_strategy: every_save
+learning_rate: 5.0e-05
+log_level: info
+logging_first_step: true
+logging_steps: 10
+logging_strategy: steps
+lora_alpha: 128
+lora_dropout: 0.05
+lora_r: 256
+lora_target_modules:
+- q_proj
+- k_proj
+- v_proj
+- o_proj
+lr_scheduler_type: cosine
+max_seq_length: 1024
+model_name_or_path: hllj/mistral-vi-math
+model_type: auto
+num_train_epochs: 2
+output_dir: outputs-sft-mistral-v1-original-data
+overwrite_output_dir: true
+per_device_eval_batch_size: 4
+per_device_train_batch_size: 4
+preprocessing_num_workers: 4
+push_to_hub: true
+report_to: wandb
+run_name: sft-mistral-v1-original-data
+save_steps: 200
+save_strategy: steps
+save_total_limit: 13
+seed: 42
+token: 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
+torch_dtype: float16
+train_file_dir: datasets/finetune_original
+use_peft: true
+validation_file_dir: datasets/validation
+warmup_ratio: 0.05
+weight_decay: 0.05

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.39,
+    "eval_loss": 0.508748471736908,
+    "eval_runtime": 6.855,
+    "eval_samples": 140,
+    "eval_samples_per_second": 20.423,
+    "eval_steps_per_second": 5.106
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.39,
+    "train_loss": 0.40742398091291976,
+    "train_runtime": 500.0224,
+    "train_samples": 1196,
+    "train_samples_per_second": 4.784,
+    "train_steps_per_second": 1.196
+}

trainer_state.json

@@ -0,0 +1,180 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.394648829431438,
+  "eval_steps": 200,
+  "global_step": 236,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.8728,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 0.8168,
+      "step": 10
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 0.6667,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 5e-05,
+      "loss": 0.5432,
+      "step": 30
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.996177016978633e-05,
+      "loss": 0.4616,
+      "step": 40
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.984719760073877e-05,
+      "loss": 0.4572,
+      "step": 50
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.9656632700046265e-05,
+      "loss": 0.4327,
+      "step": 60
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.9390658288812675e-05,
+      "loss": 0.401,
+      "step": 70
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.90500878195646e-05,
+      "loss": 0.4179,
+      "step": 80
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.8635962888399254e-05,
+      "loss": 0.4091,
+      "step": 90
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.820140360457198e-05,
+      "loss": 0.4178,
+      "step": 100
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.7651197369406566e-05,
+      "loss": 0.4046,
+      "step": 110
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 4.703171501987564e-05,
+      "loss": 0.396,
+      "step": 120
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 4.6344851172382647e-05,
+      "loss": 0.3232,
+      "step": 130
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 4.5592706521989154e-05,
+      "loss": 0.3301,
+      "step": 140
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.477758141767761e-05,
+      "loss": 0.333,
+      "step": 150
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 4.390196882699528e-05,
+      "loss": 0.3361,
+      "step": 160
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 4.296854671159614e-05,
+      "loss": 0.3169,
+      "step": 170
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 4.198016983699933e-05,
+      "loss": 0.3168,
+      "step": 180
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 4.0939861041613107e-05,
+      "loss": 0.3351,
+      "step": 190
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 3.9850801991726846e-05,
+      "loss": 0.3103,
+      "step": 200
+    },
+    {
+      "epoch": 1.27,
+      "eval_loss": 0.5224232077598572,
+      "eval_runtime": 6.8718,
+      "eval_samples_per_second": 20.373,
+      "eval_steps_per_second": 5.093,
+      "step": 200
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 3.871632345074615e-05,
+      "loss": 0.3372,
+      "step": 210
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 3.753989509243122e-05,
+      "loss": 0.3065,
+      "step": 220
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 3.632511488929382e-05,
+      "loss": 0.3254,
+      "step": 230
+    },
+    {
+      "epoch": 1.39,
+      "step": 236,
+      "total_flos": 4.241630717752115e+16,
+      "train_loss": 0.40742398091291976,
+      "train_runtime": 500.0224,
+      "train_samples_per_second": 4.784,
+      "train_steps_per_second": 1.196
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 598,
+  "num_train_epochs": 2,
+  "save_steps": 200,
+  "total_flos": 4.241630717752115e+16,
+  "trial_name": null,
+  "trial_params": null
+}