Upload folder using huggingface_hub

config.yaml

@@ -0,0 +1,79 @@
+base_model: mistralai/Mistral-7B-v0.1
+model_type: MistralForCausalLM
+tokenizer_type: LlamaTokenizer
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+datasets:
+  - path: caffeinatedcherrychic/cidds-agg-balanced
+    type: alpaca
+dataset_prepared_path: last_run_prepared
+val_set_size: 0.1
+output_dir: ./qlora-out
+
+adapter: qlora
+lora_model_dir:
+
+sequence_len: 256
+sample_packing: false
+pad_to_sequence_len: true
+
+lora_r: 32
+lora_alpha: 64
+lora_dropout: 0.05
+lora_target_linear: true
+lora_fan_in_fan_out:
+lora_target_modules:
+  - gate_proj
+  - down_proj
+  - up_proj
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+
+wandb_project:
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+gradient_accumulation_steps: 4
+micro_batch_size: 2
+num_epochs: 5
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: false
+group_by_length: false
+bf16: true
+fp16: false
+tf32: false
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+loss_watchdog_threshold: 5.0
+loss_watchdog_patience: 3
+
+max_steps: 500
+warmup_steps: 10
+evals_per_epoch: 4
+eval_table_size:
+eval_max_new_tokens: 1
+saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.001
+fsdp:
+fsdp_config:
+special_tokens:
+

dmog/axolotl-test-outputs/test.output

@@ -0,0 +1,5 @@
+Hello, dhruti
+#######
+Finetuning
+/mnt/scratch/users/dhd2000/ft14
+#######

dmog/job.error

@@ -0,0 +1,162 @@
+mpi/openmpi/4.1.5/gcc-4.8.5
+ | -- libs/gcc/system
+ |    * --> OK
+ |
+ OK
+             mpi/openmpi/4.1.5/gcc-4.8.5 ... UNLOADING --> OK
+                         libs/gcc/system ... UNLOADING --> OK
+The following values were not passed to `accelerate launch` and had defaults used instead:
+	`--num_processes` was set to a value of `1`
+	`--num_machines` was set to a value of `1`
+	`--mixed_precision` was set to a value of `'no'`
+	`--dynamo_backend` was set to a value of `'no'`
+To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
+
+
+
+
+
  0%|          | 0/62 [00:00<?, ?it/s]
  2%|▏         | 1/62 [00:03<03:15,  3.20s/it]
                                              
  2%|▏         | 1/62 [00:03<03:15,  3.20s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.95it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.32it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.47it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.07it/s]
+
                                              
+
                                             
  2%|▏         | 1/62 [00:04<03:15,  3.20s/it]
+
+
                                             
  3%|▎         | 2/62 [00:07<03:49,  3.82s/it]
                                              
  3%|▎         | 2/62 [00:07<03:49,  3.82s/it]
  5%|▍         | 3/62 [00:10<03:20,  3.40s/it]
                                              
  5%|▍         | 3/62 [00:10<03:20,  3.40s/it]
  6%|▋         | 4/62 [00:13<03:05,  3.20s/it]
                                              
  6%|▋         | 4/62 [00:13<03:05,  3.20s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.91it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.29it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.45it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.06it/s]
+
                                              
+
                                             
  6%|▋         | 4/62 [00:14<03:05,  3.20s/it]
+
+
                                             
  8%|▊         | 5/62 [00:17<03:24,  3.58s/it]
                                              
  8%|▊         | 5/62 [00:17<03:24,  3.58s/it]
 10%|▉         | 6/62 [00:20<03:07,  3.35s/it]
                                              
 10%|▉         | 6/62 [00:20<03:07,  3.35s/it]
 11%|█▏        | 7/62 [00:23<02:56,  3.20s/it]
                                              
 11%|█▏        | 7/62 [00:23<02:56,  3.20s/it]
 13%|█▎        | 8/62 [00:26<02:47,  3.10s/it]
                                              
 13%|█▎        | 8/62 [00:26<02:47,  3.10s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.92it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.28it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.44it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.04it/s]
+
                                              
+
                                             
 13%|█▎        | 8/62 [00:27<02:47,  3.10s/it]
+
+
                                             
 15%|█▍        | 9/62 [00:30<03:03,  3.47s/it]
                                              
 15%|█▍        | 9/62 [00:30<03:03,  3.47s/it]
 16%|█▌        | 10/62 [00:33<02:51,  3.29s/it]
                                               
 16%|█▌        | 10/62 [00:33<02:51,  3.29s/it]
 18%|█▊        | 11/62 [00:36<02:41,  3.17s/it]
                                               
 18%|█▊        | 11/62 [00:36<02:41,  3.17s/it]
 19%|█▉        | 12/62 [00:39<02:34,  3.09s/it]
                                               
 19%|█▉        | 12/62 [00:39<02:34,  3.09s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.89it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.27it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.43it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.03it/s]
+
                                               
+
                                             
 19%|█▉        | 12/62 [00:40<02:34,  3.09s/it]
+
+
                                             
 21%|██        | 13/62 [00:43<02:46,  3.40s/it]
                                               
 21%|██        | 13/62 [00:43<02:46,  3.40s/it]
 23%|██▎       | 14/62 [00:47<02:49,  3.53s/it]
                                               
 23%|██▎       | 14/62 [00:47<02:49,  3.53s/it]
 24%|██▍       | 15/62 [00:50<02:37,  3.35s/it]
                                               
 24%|██▍       | 15/62 [00:50<02:37,  3.35s/it]
 26%|██▌       | 16/62 [00:52<02:27,  3.21s/it]
                                               
 26%|██▌       | 16/62 [00:52<02:27,  3.21s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.86it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.24it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.41it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.02it/s]
+
                                               
+
                                             
 26%|██▌       | 16/62 [00:54<02:27,  3.21s/it]
+
+
                                             
 27%|██▋       | 17/62 [00:57<02:39,  3.54s/it]
                                               
 27%|██▋       | 17/62 [00:57<02:39,  3.54s/it]
 29%|██▉       | 18/62 [01:00<02:27,  3.35s/it]
                                               
 29%|██▉       | 18/62 [01:00<02:27,  3.35s/it]
 31%|███       | 19/62 [01:03<02:18,  3.22s/it]
                                               
 31%|███       | 19/62 [01:03<02:18,  3.22s/it]
 32%|███▏      | 20/62 [01:05<02:11,  3.13s/it]
                                               
 32%|███▏      | 20/62 [01:05<02:11,  3.13s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.83it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.24it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.41it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.01it/s]
+
                                               
+
                                             
 32%|███▏      | 20/62 [01:07<02:11,  3.13s/it]
+
+
                                             
 34%|███▍      | 21/62 [01:10<02:22,  3.48s/it]
                                               
 34%|███▍      | 21/62 [01:10<02:22,  3.48s/it]
 35%|███▌      | 22/62 [01:13<02:12,  3.31s/it]
                                               
 35%|███▌      | 22/62 [01:13<02:12,  3.31s/it]
 37%|███▋      | 23/62 [01:16<02:04,  3.19s/it]
                                               
 37%|███▋      | 23/62 [01:16<02:04,  3.19s/it]
 39%|███▊      | 24/62 [01:18<01:58,  3.11s/it]
                                               
 39%|███▊      | 24/62 [01:19<01:58,  3.11s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.83it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.23it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.39it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.00it/s]
+
                                               
+
                                             
 39%|███▊      | 24/62 [01:20<01:58,  3.11s/it]
+
+
                                             
 40%|████      | 25/62 [01:23<02:06,  3.42s/it]
                                               
 40%|████      | 25/62 [01:23<02:06,  3.42s/it]
 42%|████▏     | 26/62 [01:26<01:57,  3.27s/it]
                                               
 42%|████▏     | 26/62 [01:26<01:57,  3.27s/it]
 44%|████▎     | 27/62 [01:29<01:58,  3.39s/it]
                                               
 44%|████▎     | 27/62 [01:29<01:58,  3.39s/it]
 45%|████▌     | 28/62 [01:32<01:50,  3.25s/it]
                                               
 45%|████▌     | 28/62 [01:32<01:50,  3.25s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.84it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.23it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.40it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.00it/s]
+
                                               
+
                                             
 45%|████▌     | 28/62 [01:34<01:50,  3.25s/it]
+
+
                                             
 47%|████▋     | 29/62 [01:36<01:57,  3.57s/it]
                                               
 47%|████▋     | 29/62 [01:36<01:57,  3.57s/it]
 48%|████▊     | 30/62 [01:39<01:47,  3.37s/it]
                                               
 48%|████▊     | 30/62 [01:39<01:47,  3.37s/it]
 50%|█████     | 31/62 [01:42<01:40,  3.24s/it]
                                               
 50%|█████     | 31/62 [01:42<01:40,  3.24s/it]
 52%|█████▏    | 32/62 [01:45<01:34,  3.14s/it]
                                               
 52%|█████▏    | 32/62 [01:45<01:34,  3.14s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.81it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.21it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.39it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.00it/s]
+
                                               
+
                                             
 52%|█████▏    | 32/62 [01:47<01:34,  3.14s/it]
+
+
                                             
 53%|█████▎    | 33/62 [01:50<01:41,  3.49s/it]
                                               
 53%|█████▎    | 33/62 [01:50<01:41,  3.49s/it]
 55%|█████▍    | 34/62 [01:52<01:32,  3.32s/it]
                                               
 55%|█████▍    | 34/62 [01:52<01:32,  3.32s/it]
 56%|█████▋    | 35/62 [01:55<01:26,  3.20s/it]
                                               
 56%|█████▋    | 35/62 [01:55<01:26,  3.20s/it]
 58%|█████▊    | 36/62 [01:58<01:20,  3.11s/it]
                                               
 58%|█████▊    | 36/62 [01:58<01:20,  3.11s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.82it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.22it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.39it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.00it/s]
+
                                               
+
                                             
 58%|█████▊    | 36/62 [02:00<01:20,  3.11s/it]
+
+
                                             
 60%|█████▉    | 37/62 [02:03<01:26,  3.47s/it]
                                               
 60%|█████▉    | 37/62 [02:03<01:26,  3.47s/it]
 61%|██████▏   | 38/62 [02:05<01:18,  3.26s/it]
                                               
 61%|██████▏   | 38/62 [02:05<01:18,  3.26s/it]
 63%|██████▎   | 39/62 [02:08<01:12,  3.15s/it]
                                               
 63%|██████▎   | 39/62 [02:08<01:12,  3.15s/it]
 65%|██████▍   | 40/62 [02:12<01:12,  3.32s/it]
                                               
 65%|██████▍   | 40/62 [02:12<01:12,  3.32s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.84it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.23it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.40it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  5.00it/s]
+
                                               
+
                                             
 65%|██████▍   | 40/62 [02:13<01:12,  3.32s/it]
+
+
                                             
 66%|██████▌   | 41/62 [02:16<01:15,  3.61s/it]
                                               
 66%|██████▌   | 41/62 [02:16<01:15,  3.61s/it]
 68%|██████▊   | 42/62 [02:19<01:08,  3.40s/it]
                                               
 68%|██████▊   | 42/62 [02:19<01:08,  3.40s/it]
 69%|██████▉   | 43/62 [02:22<01:01,  3.26s/it]
                                               
 69%|██████▉   | 43/62 [02:22<01:01,  3.26s/it]
 71%|███████   | 44/62 [02:25<00:56,  3.16s/it]
                                               
 71%|███████   | 44/62 [02:25<00:56,  3.16s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.80it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.21it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.38it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  4.99it/s]
+
                                               
+
                                             
 71%|███████   | 44/62 [02:26<00:56,  3.16s/it]
+
+
                                             
 73%|███████▎  | 45/62 [02:29<00:59,  3.50s/it]
                                               
 73%|███████▎  | 45/62 [02:29<00:59,  3.50s/it]
 74%|███████▍  | 46/62 [02:32<00:53,  3.33s/it]
                                               
 74%|███████▍  | 46/62 [02:32<00:53,  3.33s/it]
 76%|███████▌  | 47/62 [02:35<00:48,  3.20s/it]
                                               
 76%|███████▌  | 47/62 [02:35<00:48,  3.20s/it]
 77%|███████▋  | 48/62 [02:38<00:43,  3.12s/it]
                                               
 77%|███████▋  | 48/62 [02:38<00:43,  3.12s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.80it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.21it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.38it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  4.98it/s]
+
                                               
+
                                             
 77%|███████▋  | 48/62 [02:39<00:43,  3.12s/it]
+
+
                                             
 79%|███████▉  | 49/62 [02:42<00:45,  3.48s/it]
                                               
 79%|███████▉  | 49/62 [02:42<00:45,  3.48s/it]
 81%|████████  | 50/62 [02:45<00:39,  3.26s/it]
                                               
 81%|████████  | 50/62 [02:45<00:39,  3.26s/it]
 82%|████████▏ | 51/62 [02:48<00:34,  3.16s/it]
                                               
 82%|████████▏ | 51/62 [02:48<00:34,  3.16s/it]
 84%|████████▍ | 52/62 [02:51<00:30,  3.09s/it]
                                               
 84%|████████▍ | 52/62 [02:51<00:30,  3.09s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.78it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.18it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.37it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  4.98it/s]
+
                                               
+
                                             
 84%|████████▍ | 52/62 [02:52<00:30,  3.09s/it]
+
+
                                             
 85%|████████▌ | 53/62 [02:56<00:33,  3.74s/it]
                                               
 85%|████████▌ | 53/62 [02:56<00:33,  3.74s/it]
 87%|████████▋ | 54/62 [02:59<00:27,  3.49s/it]
                                               
 87%|████████▋ | 54/62 [02:59<00:27,  3.49s/it]
 89%|████████▊ | 55/62 [03:02<00:23,  3.32s/it]
                                               
 89%|████████▊ | 55/62 [03:02<00:23,  3.32s/it]
 90%|█████████ | 56/62 [03:05<00:19,  3.20s/it]
                                               
 90%|█████████ | 56/62 [03:05<00:19,  3.20s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.79it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.19it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.37it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  4.99it/s]
+
                                               
+
                                             
 90%|█████████ | 56/62 [03:06<00:19,  3.20s/it]
+
+
                                             
 92%|█████████▏| 57/62 [03:09<00:17,  3.53s/it]
                                               
 92%|█████████▏| 57/62 [03:09<00:17,  3.53s/it]
 94%|█████████▎| 58/62 [03:12<00:13,  3.35s/it]
                                               
 94%|█████████▎| 58/62 [03:12<00:13,  3.35s/it]
 95%|█████████▌| 59/62 [03:15<00:09,  3.22s/it]
                                               
 95%|█████████▌| 59/62 [03:15<00:09,  3.22s/it]
 97%|█████████▋| 60/62 [03:18<00:06,  3.13s/it]
                                               
 97%|█████████▋| 60/62 [03:18<00:06,  3.13s/it]
+
  0%|          | 0/6 [00:00<?, ?it/s]
+
 33%|███▎      | 2/6 [00:00<00:00,  8.81it/s]
+
 50%|█████     | 3/6 [00:00<00:00,  6.22it/s]
+
 67%|██████▋   | 4/6 [00:00<00:00,  5.39it/s]
+
 83%|████████▎ | 5/6 [00:00<00:00,  4.99it/s]
+
                                               
+
                                             
 97%|█████████▋| 60/62 [03:19<00:06,  3.13s/it]
+
+
                                             
 98%|█████████▊| 61/62 [03:22<00:03,  3.49s/it]
                                               
 98%|█████████▊| 61/62 [03:22<00:03,  3.49s/it]
                                               
                                               

dmog/job.output

@@ -0,0 +1,131 @@
+Your results will be stored in: /mnt/scratch/users/dhd2000/ft14/dmog/axolotl-test-outputs
+Executing job commands, current working directory is /mnt/scratch/users/dhd2000/ft14/dmog
+/mnt/scratch/users/dhd2000/ft14
+[2024-04-09 08:29:06,912] [INFO] [datasets.<module>:58] [PID:30736] PyTorch version 2.1.2 available.
+[2024-04-09 08:29:08,482] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2024-04-09 08:29:10,330] [INFO] [axolotl.normalize_config:178] [PID:30736] [RANK:0] GPU memory usage baseline: 0.000GB (+0.640GB misc)
+                                 dP            dP   dP 
+                                 88            88   88 
+      .d8888b. dP.  .dP .d8888b. 88 .d8888b. d8888P 88 
+      88'  `88  `8bd8'  88'  `88 88 88'  `88   88   88 
+      88.  .88  .d88b.  88.  .88 88 88.  .88   88   88 
+      `88888P8 dP'  `dP `88888P' dP `88888P'   dP   dP 
+                                                       
+                                                       
+
+[2024-04-09 08:29:10,707] [DEBUG] [axolotl.load_tokenizer:245] [PID:30736] [RANK:0] EOS: 2 / </s>
+[2024-04-09 08:29:10,708] [DEBUG] [axolotl.load_tokenizer:246] [PID:30736] [RANK:0] BOS: 1 / <s>
+[2024-04-09 08:29:10,708] [DEBUG] [axolotl.load_tokenizer:247] [PID:30736] [RANK:0] PAD: 2 / </s>
+[2024-04-09 08:29:10,708] [DEBUG] [axolotl.load_tokenizer:248] [PID:30736] [RANK:0] UNK: 0 / <unk>
+[2024-04-09 08:29:10,708] [INFO] [axolotl.load_tokenizer:259] [PID:30736] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.
+[2024-04-09 08:29:10,708] [INFO] [axolotl.load_tokenized_prepared_datasets:191] [PID:30736] [RANK:0] Unable to find prepared dataset in last_run_prepared/c62a45f4c109d3ed049f89406e7ab97e
+[2024-04-09 08:29:10,708] [INFO] [axolotl.load_tokenized_prepared_datasets:192] [PID:30736] [RANK:0] Loading raw datasets...
+[2024-04-09 08:29:10,709] [WARNING] [axolotl.load_tokenized_prepared_datasets:194] [PID:30736] [RANK:0] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset.
+[2024-04-09 08:29:10,709] [INFO] [axolotl.load_tokenized_prepared_datasets:201] [PID:30736] [RANK:0] No seed provided, using default seed of 42
+[2024-04-09 08:29:17,092] [INFO] [axolotl.load_tokenized_prepared_datasets:414] [PID:30736] [RANK:0] merging datasets
+[2024-04-09 08:29:17,096] [INFO] [axolotl.log:61] [PID:30736] [RANK:0] dropping attention_mask column
+[2024-04-09 08:29:18,698] [INFO] [axolotl.load_tokenized_prepared_datasets:424] [PID:30736] [RANK:0] Saving merged prepared dataset to disk... last_run_prepared/c62a45f4c109d3ed049f89406e7ab97e
+[2024-04-09 08:29:18,755] [DEBUG] [axolotl.log:61] [PID:30736] [RANK:0] total_num_tokens: 21468
+[2024-04-09 08:29:18,756] [DEBUG] [axolotl.log:61] [PID:30736] [RANK:0] `total_supervised_tokens: 259`
+[2024-04-09 08:29:18,756] [DEBUG] [axolotl.log:61] [PID:30736] [RANK:0] total_num_steps: 62
+[2024-04-09 08:29:18,756] [INFO] [axolotl.prepare_dataset:124] [PID:30736] [RANK:0] Maximum number of steps set at 62
+[2024-04-09 08:29:18,759] [DEBUG] [axolotl.train.log:61] [PID:30736] [RANK:0] loading tokenizer... mistralai/Mistral-7B-v0.1
+[2024-04-09 08:29:19,028] [DEBUG] [axolotl.load_tokenizer:245] [PID:30736] [RANK:0] EOS: 2 / </s>
+[2024-04-09 08:29:19,028] [DEBUG] [axolotl.load_tokenizer:246] [PID:30736] [RANK:0] BOS: 1 / <s>
+[2024-04-09 08:29:19,028] [DEBUG] [axolotl.load_tokenizer:247] [PID:30736] [RANK:0] PAD: 2 / </s>
+[2024-04-09 08:29:19,029] [DEBUG] [axolotl.load_tokenizer:248] [PID:30736] [RANK:0] UNK: 0 / <unk>
+[2024-04-09 08:29:19,029] [INFO] [axolotl.load_tokenizer:259] [PID:30736] [RANK:0] No Chat template selected. Consider adding a chat template for easier inference.
+[2024-04-09 08:29:19,029] [DEBUG] [axolotl.train.log:61] [PID:30736] [RANK:0] loading model and peft_config...
+[2024-04-09 08:29:35,702] [INFO] [axolotl.load_model:660] [PID:30736] [RANK:0] GPU memory usage after model load: 4.342GB (+0.138GB cache, +0.942GB misc)
+[2024-04-09 08:29:35,711] [INFO] [axolotl.load_model:701] [PID:30736] [RANK:0] converting PEFT model w/ prepare_model_for_kbit_training
+[2024-04-09 08:29:35,713] [INFO] [axolotl.load_model:710] [PID:30736] [RANK:0] converting modules to torch.bfloat16 for flash attention
+[2024-04-09 08:29:35,715] [INFO] [axolotl.load_lora:825] [PID:30736] [RANK:0] found linear modules: ['up_proj', 'q_proj', 'k_proj', 'gate_proj', 'down_proj', 'o_proj', 'v_proj']
+trainable params: 83,886,080 || all params: 7,325,618,176 || trainable%: 1.1451058188485088
+[2024-04-09 08:29:36,348] [INFO] [axolotl.load_model:750] [PID:30736] [RANK:0] GPU memory usage after adapters: 4.670GB (+0.935GB cache, +0.942GB misc)
+[2024-04-09 08:29:36,446] [INFO] [axolotl.train.log:61] [PID:30736] [RANK:0] Pre-saving adapter config to ./qlora-out
+[2024-04-09 08:29:36,459] [INFO] [axolotl.train.log:61] [PID:30736] [RANK:0] Starting trainer...
+{'loss': 6.6367, 'grad_norm': 102.28898620605469, 'learning_rate': 2e-05, 'epoch': 0.08}
+{'eval_loss': 7.300913333892822, 'eval_runtime': 1.3523, 'eval_samples_per_second': 8.873, 'eval_steps_per_second': 4.437, 'epoch': 0.08}
+[2024-04-09 08:29:44,573] [INFO] [axolotl.callbacks.on_step_end:123] [PID:30736] [RANK:0] GPU memory usage while training: 4.843GB (+1.177GB cache, +0.965GB misc)
+{'loss': 7.0616, 'grad_norm': 103.4541015625, 'learning_rate': 4e-05, 'epoch': 0.16}
+{'loss': 4.686, 'grad_norm': 67.47515869140625, 'learning_rate': 6e-05, 'epoch': 0.24}
+{'loss': 2.3866, 'grad_norm': 72.36919403076172, 'learning_rate': 8e-05, 'epoch': 0.32}
+{'eval_loss': 0.7137572169303894, 'eval_runtime': 1.3532, 'eval_samples_per_second': 8.868, 'eval_steps_per_second': 4.434, 'epoch': 0.32}
+{'loss': 0.6844, 'grad_norm': 16.83085060119629, 'learning_rate': 0.0001, 'epoch': 0.4}
+{'loss': 0.914, 'grad_norm': 25.897714614868164, 'learning_rate': 0.00012, 'epoch': 0.48}
+{'loss': 0.63, 'grad_norm': 18.89151382446289, 'learning_rate': 0.00014, 'epoch': 0.56}
+{'loss': 0.948, 'grad_norm': 27.15555763244629, 'learning_rate': 0.00016, 'epoch': 0.64}
+{'eval_loss': 1.0445994138717651, 'eval_runtime': 1.356, 'eval_samples_per_second': 8.85, 'eval_steps_per_second': 4.425, 'epoch': 0.64}
+{'loss': 1.0285, 'grad_norm': 20.812381744384766, 'learning_rate': 0.00018, 'epoch': 0.72}
+{'loss': 1.3756, 'grad_norm': 56.3886604309082, 'learning_rate': 0.0002, 'epoch': 0.8}
+{'loss': 0.5178, 'grad_norm': 6.24803352355957, 'learning_rate': 0.00019981755542233177, 'epoch': 0.88}
+{'loss': 0.6822, 'grad_norm': 8.379430770874023, 'learning_rate': 0.0001992708874098054, 'epoch': 0.96}
+{'eval_loss': 1.3959709405899048, 'eval_runtime': 1.3583, 'eval_samples_per_second': 8.835, 'eval_steps_per_second': 4.417, 'epoch': 0.96}
+{'loss': 1.3762, 'grad_norm': 20.744348526000977, 'learning_rate': 0.00019836199069471437, 'epoch': 1.04}
+{'loss': 0.5248, 'grad_norm': 4.800480842590332, 'learning_rate': 0.0001970941817426052, 'epoch': 1.12}
+{'loss': 0.8094, 'grad_norm': 11.284302711486816, 'learning_rate': 0.00019547208665085457, 'epoch': 1.2}
+{'loss': 0.5222, 'grad_norm': 5.787976264953613, 'learning_rate': 0.0001935016242685415, 'epoch': 1.28}
+{'eval_loss': 0.9023411870002747, 'eval_runtime': 1.3623, 'eval_samples_per_second': 8.808, 'eval_steps_per_second': 4.404, 'epoch': 1.28}
+{'loss': 0.8027, 'grad_norm': 21.48629379272461, 'learning_rate': 0.00019118998459920902, 'epoch': 1.36}
+{'loss': 1.7772, 'grad_norm': 38.0982666015625, 'learning_rate': 0.000188545602565321, 'epoch': 1.44}
+{'loss': 0.7737, 'grad_norm': 10.824837684631348, 'learning_rate': 0.00018557812723014476, 'epoch': 1.52}
+{'loss': 0.534, 'grad_norm': 9.1353120803833, 'learning_rate': 0.00018229838658936564, 'epoch': 1.6}
+{'eval_loss': 0.4847445785999298, 'eval_runtime': 1.3637, 'eval_samples_per_second': 8.799, 'eval_steps_per_second': 4.4, 'epoch': 1.6}
+{'loss': 0.3201, 'grad_norm': 3.8411033153533936, 'learning_rate': 0.00017871834806090501, 'epoch': 1.68}
+{'loss': 2.2541, 'grad_norm': 23.888507843017578, 'learning_rate': 0.00017485107481711012, 'epoch': 1.76}
+{'loss': 0.8177, 'grad_norm': 8.5956392288208, 'learning_rate': 0.00017071067811865476, 'epoch': 1.84}
+{'loss': 0.4624, 'grad_norm': 3.825141191482544, 'learning_rate': 0.00016631226582407952, 'epoch': 1.92}
+{'eval_loss': 0.5740255117416382, 'eval_runtime': 1.3655, 'eval_samples_per_second': 8.788, 'eval_steps_per_second': 4.394, 'epoch': 1.92}
+{'loss': 0.3714, 'grad_norm': 3.558993101119995, 'learning_rate': 0.00016167188726285434, 'epoch': 2.0}
+{'loss': 0.6562, 'grad_norm': 11.759211540222168, 'learning_rate': 0.00015680647467311557, 'epoch': 2.08}
+{'loss': 1.5141, 'grad_norm': 96.2179183959961, 'learning_rate': 0.00015173378141776568, 'epoch': 2.16}
+{'loss': 0.7753, 'grad_norm': 31.022045135498047, 'learning_rate': 0.00014647231720437686, 'epoch': 2.24}
+{'eval_loss': 0.3771994113922119, 'eval_runtime': 1.3676, 'eval_samples_per_second': 8.775, 'eval_steps_per_second': 4.387, 'epoch': 2.24}
+{'loss': 0.2649, 'grad_norm': 3.5004501342773438, 'learning_rate': 0.0001410412805452757, 'epoch': 2.32}
+{'loss': 0.171, 'grad_norm': 5.16464376449585, 'learning_rate': 0.00013546048870425356, 'epoch': 2.4}
+{'loss': 0.9172, 'grad_norm': 25.634010314941406, 'learning_rate': 0.00012975030538552032, 'epoch': 2.48}
+{'loss': 0.3324, 'grad_norm': 7.102908134460449, 'learning_rate': 0.0001239315664287558, 'epoch': 2.56}
+{'eval_loss': 0.29374203085899353, 'eval_runtime': 1.3678, 'eval_samples_per_second': 8.773, 'eval_steps_per_second': 4.387, 'epoch': 2.56}
+{'loss': 0.4932, 'grad_norm': 6.236325263977051, 'learning_rate': 0.0001180255037813906, 'epoch': 2.64}
+{'loss': 0.1284, 'grad_norm': 4.445058345794678, 'learning_rate': 0.0001120536680255323, 'epoch': 2.72}
+{'loss': 0.1547, 'grad_norm': 6.94170618057251, 'learning_rate': 0.00010603784974222861, 'epoch': 2.8}
+{'loss': 0.1973, 'grad_norm': 5.656033039093018, 'learning_rate': 0.0001, 'epoch': 2.88}
+{'eval_loss': 0.5674905180931091, 'eval_runtime': 1.3681, 'eval_samples_per_second': 8.771, 'eval_steps_per_second': 4.386, 'epoch': 2.88}
+{'loss': 0.4884, 'grad_norm': 18.19667625427246, 'learning_rate': 9.396215025777139e-05, 'epoch': 2.96}
+{'loss': 0.5526, 'grad_norm': 17.964893341064453, 'learning_rate': 8.79463319744677e-05, 'epoch': 3.04}
+{'loss': 0.2116, 'grad_norm': 5.015590190887451, 'learning_rate': 8.197449621860943e-05, 'epoch': 3.12}
+{'loss': 0.0843, 'grad_norm': 5.6883225440979, 'learning_rate': 7.606843357124426e-05, 'epoch': 3.2}
+{'eval_loss': 0.2360386848449707, 'eval_runtime': 1.3667, 'eval_samples_per_second': 8.78, 'eval_steps_per_second': 4.39, 'epoch': 3.2}
+{'loss': 0.1158, 'grad_norm': 6.636446475982666, 'learning_rate': 7.024969461447972e-05, 'epoch': 3.28}
+{'loss': 0.2755, 'grad_norm': 4.405576229095459, 'learning_rate': 6.453951129574644e-05, 'epoch': 3.36}
+{'loss': 0.0186, 'grad_norm': 1.6179524660110474, 'learning_rate': 5.8958719454724346e-05, 'epoch': 3.44}
+{'loss': 0.3836, 'grad_norm': 8.783114433288574, 'learning_rate': 5.3527682795623146e-05, 'epoch': 3.52}
+{'eval_loss': 0.13969357311725616, 'eval_runtime': 1.3687, 'eval_samples_per_second': 8.767, 'eval_steps_per_second': 4.384, 'epoch': 3.52}
+{'loss': 0.0141, 'grad_norm': 0.8835445046424866, 'learning_rate': 4.826621858223431e-05, 'epoch': 3.6}
+{'loss': 0.6196, 'grad_norm': 12.678099632263184, 'learning_rate': 4.3193525326884435e-05, 'epoch': 3.68}
+{'loss': 0.0948, 'grad_norm': 5.320870876312256, 'learning_rate': 3.832811273714569e-05, 'epoch': 3.76}
+{'loss': 0.0449, 'grad_norm': 2.7501108646392822, 'learning_rate': 3.36877341759205e-05, 'epoch': 3.84}
+{'eval_loss': 0.2801015079021454, 'eval_runtime': 1.3706, 'eval_samples_per_second': 8.755, 'eval_steps_per_second': 4.378, 'epoch': 3.84}
+{'loss': 0.3026, 'grad_norm': 4.41072940826416, 'learning_rate': 2.9289321881345254e-05, 'epoch': 3.92}
+{'loss': 0.0152, 'grad_norm': 1.2105910778045654, 'learning_rate': 2.514892518288988e-05, 'epoch': 4.0}
+{'loss': 0.0629, 'grad_norm': 4.502895355224609, 'learning_rate': 2.1281651939094992e-05, 'epoch': 4.08}
+{'loss': 0.2246, 'grad_norm': 6.058006286621094, 'learning_rate': 1.7701613410634365e-05, 'epoch': 4.16}
+{'eval_loss': 0.19463467597961426, 'eval_runtime': 1.3725, 'eval_samples_per_second': 8.743, 'eval_steps_per_second': 4.372, 'epoch': 4.16}
+{'loss': 0.0093, 'grad_norm': 0.5118169784545898, 'learning_rate': 1.442187276985526e-05, 'epoch': 4.24}
+{'loss': 0.0148, 'grad_norm': 0.8497004508972168, 'learning_rate': 1.1454397434679021e-05, 'epoch': 4.32}
+{'loss': 0.0392, 'grad_norm': 1.752151608467102, 'learning_rate': 8.810015400790994e-06, 'epoch': 4.4}
+{'loss': 0.229, 'grad_norm': 3.6673429012298584, 'learning_rate': 6.498375731458528e-06, 'epoch': 4.48}
+{'eval_loss': 0.16181980073451996, 'eval_runtime': 1.3705, 'eval_samples_per_second': 8.756, 'eval_steps_per_second': 4.378, 'epoch': 4.48}
+{'loss': 0.1722, 'grad_norm': 2.9522616863250732, 'learning_rate': 4.527913349145441e-06, 'epoch': 4.56}
+{'loss': 0.0295, 'grad_norm': 1.5037487745285034, 'learning_rate': 2.905818257394799e-06, 'epoch': 4.64}
+{'loss': 0.03, 'grad_norm': 1.4181660413742065, 'learning_rate': 1.6380093052856483e-06, 'epoch': 4.72}
+{'loss': 0.3073, 'grad_norm': 9.207091331481934, 'learning_rate': 7.291125901946027e-07, 'epoch': 4.8}
+{'eval_loss': 0.14654164016246796, 'eval_runtime': 1.3704, 'eval_samples_per_second': 8.756, 'eval_steps_per_second': 4.378, 'epoch': 4.8}
+{'loss': 0.032, 'grad_norm': 1.5023337602615356, 'learning_rate': 1.824445776682504e-07, 'epoch': 4.88}
+{'loss': 0.1144, 'grad_norm': 2.882874011993408, 'learning_rate': 0.0, 'epoch': 4.96}
+{'train_runtime': 206.4235, 'train_samples_per_second': 2.403, 'train_steps_per_second': 0.3, 'train_loss': 0.7901421915739775, 'epoch': 4.96}
+[2024-04-09 08:33:03,093] [INFO] [axolotl.train.log:61] [PID:30736] [RANK:0] Training Completed!!! Saving pre-trained model to ./qlora-out
+(PeftModelForCausalLM(   (base_model): LoraModel(     (model): MistralForCausalLM(       (model): MistralModel(         (embed_tokens): Embedding(32000, 4096)         (layers): ModuleList(           (0-31): 32 x MistralDecoderLayer(             (self_attn): MistralFlashAttention2(               (q_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=4096, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (k_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=1024, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (v_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=1024, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (o_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=4096, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (rotary_emb): MistralRotaryEmbedding()             )             (mlp): MistralMLP(               (gate_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=14336, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=14336, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (up_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=4096, out_features=14336, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=4096, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=14336, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (down_proj): lora.Linear4bit(                 (base_layer): Linear4bit(in_features=14336, out_features=4096, bias=False)                 (lora_dropout): ModuleDict(                   (default): Dropout(p=0.05, inplace=False)                 )                 (lora_A): ModuleDict(                   (default): Linear(in_features=14336, out_features=32, bias=False)                 )                 (lora_B): ModuleDict(                   (default): Linear(in_features=32, out_features=4096, bias=False)                 )                 (lora_embedding_A): ParameterDict()                 (lora_embedding_B): ParameterDict()               )               (act_fn): SiLU()             )             (input_layernorm): MistralRMSNorm()             (post_attention_layernorm): MistralRMSNorm()           )         )         (norm): MistralRMSNorm()       )       (lm_head): Linear(in_features=4096, out_features=32000, bias=False)     )   ) ), LlamaTokenizer(name_or_path='mistralai/Mistral-7B-v0.1', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '</s>'}, clean_up_tokenization_spaces=False),  added_tokens_decoder={ 	0: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), 	1: AddedToken("<s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), 	2: AddedToken("</s>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), })
+
+End of job
+Output file has been generated, please check /mnt/scratch/users/dhd2000/ft14/dmog/axolotl-test-outputs/test.output

finetune-test.py

@@ -0,0 +1,72 @@
+# This script is used to test the model using a dataset 
+
+# Import the necessary libraries
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from langchain.memory import ConversationBufferWindowMemory
+from peft import PeftModel
+import torch
+
+import json
+import sys
+
+# Check if the correct number of arguments are provided
+if len(sys.argv) != 2:
+    print("Usage: python finetune.py <jsonl_file>")
+    sys.exit(1)
+
+# Get the file path from the command-line argument
+jsonl_file_path = sys.argv[1]
+
+# Load the model and tokenizer
+base_model = "mistralai/Mistral-7B-Instruct-v0.2"
+tokenizer = AutoTokenizer.from_pretrained(base_model)
+tokenizer.add_special_tokens({"pad_token": "[PAD]"})
+base_model = AutoModelForCausalLM.from_pretrained(base_model)
+
+
+ft_model = PeftModel.from_pretrained(base_model, "./qlora-out") 
+# ft_model = ft_model.merge_and_unload()
+ft_model.eval()
+
+# Set the device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ft_model.to(device)
+
+# Read the JSONL file
+with open(jsonl_file_path, "r") as f:
+    tp, tn, fp, fn = 0, 0, 0, 0
+    for line in f:
+        data = json.loads(line)
+        user_in = data["input"]
+        user_input = f"[INST] ###instruction: Check if the given traffic flow is normal or of an attacker or a victim\n###input: {user_in}\n#output: [/INST]"
+        encodings = tokenizer(user_input, return_tensors="pt", padding=True).to(device)
+        input_ids = encodings["input_ids"]
+        attention_mask = encodings["attention_mask"]
+
+        output_ids = ft_model.generate(input_ids, attention_mask = attention_mask, max_new_tokens=1000, num_return_sequences=1, do_sample=True, temperature=0.1, top_p=0.9)
+
+        generated_ids = output_ids[0, input_ids.shape[-1]:]
+
+        # Decode the output
+        response = tokenizer.decode(generated_ids, skip_special_tokens=True).lower()
+
+        # calculate true positive, true negative, false positive, false negative
+        if "normal" not in response and data["output"] == response:
+            tp += 1
+        elif "normal" in response and data["output"] == response:
+            tn += 1
+        elif "normal" in response and data["output"] != response:
+            fp += 1
+        elif "normal" not in response and data["output"] != response:
+            fn += 1
+        else: 
+            print(f"Error: {response}, {data[output]}")
+            print(f"User input: {user_in}")
+            print(f"Generated response: {response}")
+            print(f"Expected response: {data[output]}")
+            print()
+
+        print(f"TP: {tp}, TN: {tn}, FP: {fp}, FN: {fn}")
+            
+
+

last_run_prepared/c62a45f4c109d3ed049f89406e7ab97e/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a8a6b98277d114990fb441a27d2f777773005e6b7cf57a0ec219fe3bae40b1
+size 259336

last_run_prepared/c62a45f4c109d3ed049f89406e7ab97e/dataset_info.json

@@ -0,0 +1,22 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "labels": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

last_run_prepared/c62a45f4c109d3ed049f89406e7ab97e/state.json

@@ -0,0 +1,16 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "992b9317aa372e8e",
+  "_format_columns": [
+    "input_ids",
+    "labels"
+  ],
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///mnt/scratch/users/dhd2000/ft14/mlruns/0/7e75ece8e18e485db64e4e2d9196e738/artifacts
+end_time: 1712647983089
+entry_point_name: ''
+experiment_id: '0'
+lifecycle_stage: active
+run_id: 7e75ece8e18e485db64e4e2d9196e738
+run_name: ./qlora-out
+run_uuid: 7e75ece8e18e485db64e4e2d9196e738
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1712647776681
+status: 3
+tags: []
+user_id: dhd2000

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/epoch

@@ -0,0 +1,79 @@
+1712647780322 0.08 1
+1712647781687 0.08 1
+1712647784574 0.16 2
+1712647787465 0.24 3
+1712647790359 0.32 4
+1712647791728 0.32 4
+1712647794625 0.4 5
+1712647797520 0.48 6
+1712647800416 0.56 7
+1712647803312 0.64 8
+1712647804683 0.64 8
+1712647807578 0.72 9
+1712647810474 0.8 10
+1712647813372 0.88 11
+1712647816270 0.96 12
+1712647817642 0.96 12
+1712647820386 1.04 13
+1712647824221 1.12 14
+1712647827138 1.2 15
+1712647830042 1.28 16
+1712647831420 1.28 16
+1712647834333 1.36 17
+1712647837242 1.44 18
+1712647840147 1.52 19
+1712647843070 1.6 20
+1712647844448 1.6 20
+1712647847363 1.68 21
+1712647850290 1.76 22
+1712647853203 1.84 23
+1712647856116 1.92 24
+1712647857496 1.92 24
+1712647860252 2.0 25
+1712647863163 2.08 26
+1712647866855 2.16 27
+1712647869769 2.24 28
+1712647871150 2.24 28
+1712647874087 2.32 29
+1712647877006 2.4 30
+1712647879921 2.48 31
+1712647882836 2.56 32
+1712647884219 2.56 32
+1712647887139 2.64 33
+1712647890053 2.72 34
+1712647892967 2.8 35
+1712647895882 2.88 36
+1712647897265 2.88 36
+1712647900187 2.96 37
+1712647902945 3.04 38
+1712647905861 3.12 39
+1712647909552 3.2 40
+1712647910933 3.2 40
+1712647913855 3.28 41
+1712647916777 3.36 42
+1712647919694 3.44 43
+1712647922609 3.52 44
+1712647923992 3.52 44
+1712647926911 3.6 45
+1712647929833 3.68 46
+1712647932754 3.76 47
+1712647935680 3.84 48
+1712647937065 3.84 48
+1712647939993 3.92 49
+1712647942758 4.0 50
+1712647945681 4.08 51
+1712647948606 4.16 52
+1712647949993 4.16 52
+1712647953856 4.24 53
+1712647956779 4.32 54
+1712647959701 4.4 55
+1712647962622 4.48 56
+1712647964007 4.48 56
+1712647966930 4.56 57
+1712647969855 4.64 58
+1712647972779 4.72 59
+1712647975702 4.8 60
+1712647977087 4.8 60
+1712647980013 4.88 61
+1712647982936 4.96 62
+1712647983084 4.96 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/eval_loss

@@ -0,0 +1,16 @@
+1712647781687 7.300913333892822 1
+1712647791728 0.7137572169303894 4
+1712647804683 1.0445994138717651 8
+1712647817642 1.3959709405899048 12
+1712647831420 0.9023411870002747 16
+1712647844448 0.4847445785999298 20
+1712647857496 0.5740255117416382 24
+1712647871150 0.3771994113922119 28
+1712647884219 0.29374203085899353 32
+1712647897265 0.5674905180931091 36
+1712647910933 0.2360386848449707 40
+1712647923992 0.13969357311725616 44
+1712647937065 0.2801015079021454 48
+1712647949993 0.19463467597961426 52
+1712647964007 0.16181980073451996 56
+1712647977087 0.14654164016246796 60

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/eval_runtime

@@ -0,0 +1,16 @@
+1712647781687 1.3523 1
+1712647791728 1.3532 4
+1712647804683 1.356 8
+1712647817642 1.3583 12
+1712647831420 1.3623 16
+1712647844448 1.3637 20
+1712647857496 1.3655 24
+1712647871150 1.3676 28
+1712647884219 1.3678 32
+1712647897265 1.3681 36
+1712647910933 1.3667 40
+1712647923992 1.3687 44
+1712647937065 1.3706 48
+1712647949993 1.3725 52
+1712647964007 1.3705 56
+1712647977087 1.3704 60

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/eval_samples_per_second

@@ -0,0 +1,16 @@
+1712647781687 8.873 1
+1712647791728 8.868 4
+1712647804683 8.85 8
+1712647817642 8.835 12
+1712647831420 8.808 16
+1712647844448 8.799 20
+1712647857496 8.788 24
+1712647871150 8.775 28
+1712647884219 8.773 32
+1712647897265 8.771 36
+1712647910933 8.78 40
+1712647923992 8.767 44
+1712647937065 8.755 48
+1712647949993 8.743 52
+1712647964007 8.756 56
+1712647977087 8.756 60

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/eval_steps_per_second

@@ -0,0 +1,16 @@
+1712647781687 4.437 1
+1712647791728 4.434 4
+1712647804683 4.425 8
+1712647817642 4.417 12
+1712647831420 4.404 16
+1712647844448 4.4 20
+1712647857496 4.394 24
+1712647871150 4.387 28
+1712647884219 4.387 32
+1712647897265 4.386 36
+1712647910933 4.39 40
+1712647923992 4.384 44
+1712647937065 4.378 48
+1712647949993 4.372 52
+1712647964007 4.378 56
+1712647977087 4.378 60

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/grad_norm

@@ -0,0 +1,62 @@
+1712647780322 102.28898620605469 1
+1712647784574 103.4541015625 2
+1712647787465 67.47515869140625 3
+1712647790359 72.36919403076172 4
+1712647794625 16.83085060119629 5
+1712647797520 25.897714614868164 6
+1712647800416 18.89151382446289 7
+1712647803312 27.15555763244629 8
+1712647807578 20.812381744384766 9
+1712647810474 56.3886604309082 10
+1712647813372 6.24803352355957 11
+1712647816270 8.379430770874023 12
+1712647820386 20.744348526000977 13
+1712647824221 4.800480842590332 14
+1712647827138 11.284302711486816 15
+1712647830042 5.787976264953613 16
+1712647834333 21.48629379272461 17
+1712647837242 38.0982666015625 18
+1712647840147 10.824837684631348 19
+1712647843070 9.1353120803833 20
+1712647847363 3.8411033153533936 21
+1712647850290 23.888507843017578 22
+1712647853203 8.5956392288208 23
+1712647856116 3.825141191482544 24
+1712647860252 3.558993101119995 25
+1712647863163 11.759211540222168 26
+1712647866855 96.2179183959961 27
+1712647869769 31.022045135498047 28
+1712647874087 3.5004501342773438 29
+1712647877006 5.16464376449585 30
+1712647879921 25.634010314941406 31
+1712647882836 7.102908134460449 32
+1712647887139 6.236325263977051 33
+1712647890053 4.445058345794678 34
+1712647892967 6.94170618057251 35
+1712647895882 5.656033039093018 36
+1712647900187 18.19667625427246 37
+1712647902945 17.964893341064453 38
+1712647905861 5.015590190887451 39
+1712647909552 5.6883225440979 40
+1712647913855 6.636446475982666 41
+1712647916777 4.405576229095459 42
+1712647919694 1.6179524660110474 43
+1712647922609 8.783114433288574 44
+1712647926911 0.8835445046424866 45
+1712647929833 12.678099632263184 46
+1712647932754 5.320870876312256 47
+1712647935680 2.7501108646392822 48
+1712647939993 4.41072940826416 49
+1712647942758 1.2105910778045654 50
+1712647945681 4.502895355224609 51
+1712647948606 6.058006286621094 52
+1712647953856 0.5118169784545898 53
+1712647956779 0.8497004508972168 54
+1712647959701 1.752151608467102 55
+1712647962622 3.6673429012298584 56
+1712647966930 2.9522616863250732 57
+1712647969855 1.5037487745285034 58
+1712647972779 1.4181660413742065 59
+1712647975702 9.207091331481934 60
+1712647980013 1.5023337602615356 61
+1712647982936 2.882874011993408 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/learning_rate

@@ -0,0 +1,62 @@
+1712647780322 2e-05 1
+1712647784574 4e-05 2
+1712647787465 6e-05 3
+1712647790359 8e-05 4
+1712647794625 0.0001 5
+1712647797520 0.00012 6
+1712647800416 0.00014 7
+1712647803312 0.00016 8
+1712647807578 0.00018 9
+1712647810474 0.0002 10
+1712647813372 0.00019981755542233177 11
+1712647816270 0.0001992708874098054 12
+1712647820386 0.00019836199069471437 13
+1712647824221 0.0001970941817426052 14
+1712647827138 0.00019547208665085457 15
+1712647830042 0.0001935016242685415 16
+1712647834333 0.00019118998459920902 17
+1712647837242 0.000188545602565321 18
+1712647840147 0.00018557812723014476 19
+1712647843070 0.00018229838658936564 20
+1712647847363 0.00017871834806090501 21
+1712647850290 0.00017485107481711012 22
+1712647853203 0.00017071067811865476 23
+1712647856116 0.00016631226582407952 24
+1712647860252 0.00016167188726285434 25
+1712647863163 0.00015680647467311557 26
+1712647866855 0.00015173378141776568 27
+1712647869769 0.00014647231720437686 28
+1712647874087 0.0001410412805452757 29
+1712647877006 0.00013546048870425356 30
+1712647879921 0.00012975030538552032 31
+1712647882836 0.0001239315664287558 32
+1712647887139 0.0001180255037813906 33
+1712647890053 0.0001120536680255323 34
+1712647892967 0.00010603784974222861 35
+1712647895882 0.0001 36
+1712647900187 9.396215025777139e-05 37
+1712647902945 8.79463319744677e-05 38
+1712647905861 8.197449621860943e-05 39
+1712647909552 7.606843357124426e-05 40
+1712647913855 7.024969461447972e-05 41
+1712647916777 6.453951129574644e-05 42
+1712647919694 5.8958719454724346e-05 43
+1712647922609 5.3527682795623146e-05 44
+1712647926911 4.826621858223431e-05 45
+1712647929833 4.3193525326884435e-05 46
+1712647932754 3.832811273714569e-05 47
+1712647935680 3.36877341759205e-05 48
+1712647939993 2.9289321881345254e-05 49
+1712647942758 2.514892518288988e-05 50
+1712647945681 2.1281651939094992e-05 51
+1712647948606 1.7701613410634365e-05 52
+1712647953856 1.442187276985526e-05 53
+1712647956779 1.1454397434679021e-05 54
+1712647959701 8.810015400790994e-06 55
+1712647962622 6.498375731458528e-06 56
+1712647966930 4.527913349145441e-06 57
+1712647969855 2.905818257394799e-06 58
+1712647972779 1.6380093052856483e-06 59
+1712647975702 7.291125901946027e-07 60
+1712647980013 1.824445776682504e-07 61
+1712647982936 0.0 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/loss

@@ -0,0 +1,62 @@
+1712647780322 6.6367 1
+1712647784574 7.0616 2
+1712647787465 4.686 3
+1712647790359 2.3866 4
+1712647794625 0.6844 5
+1712647797520 0.914 6
+1712647800416 0.63 7
+1712647803312 0.948 8
+1712647807578 1.0285 9
+1712647810474 1.3756 10
+1712647813372 0.5178 11
+1712647816270 0.6822 12
+1712647820386 1.3762 13
+1712647824221 0.5248 14
+1712647827138 0.8094 15
+1712647830042 0.5222 16
+1712647834333 0.8027 17
+1712647837242 1.7772 18
+1712647840147 0.7737 19
+1712647843070 0.534 20
+1712647847363 0.3201 21
+1712647850290 2.2541 22
+1712647853203 0.8177 23
+1712647856116 0.4624 24
+1712647860252 0.3714 25
+1712647863163 0.6562 26
+1712647866855 1.5141 27
+1712647869769 0.7753 28
+1712647874087 0.2649 29
+1712647877006 0.171 30
+1712647879921 0.9172 31
+1712647882836 0.3324 32
+1712647887139 0.4932 33
+1712647890053 0.1284 34
+1712647892967 0.1547 35
+1712647895882 0.1973 36
+1712647900187 0.4884 37
+1712647902945 0.5526 38
+1712647905861 0.2116 39
+1712647909552 0.0843 40
+1712647913855 0.1158 41
+1712647916777 0.2755 42
+1712647919694 0.0186 43
+1712647922609 0.3836 44
+1712647926911 0.0141 45
+1712647929833 0.6196 46
+1712647932754 0.0948 47
+1712647935680 0.0449 48
+1712647939993 0.3026 49
+1712647942758 0.0152 50
+1712647945681 0.0629 51
+1712647948606 0.2246 52
+1712647953856 0.0093 53
+1712647956779 0.0148 54
+1712647959701 0.0392 55
+1712647962622 0.229 56
+1712647966930 0.1722 57
+1712647969855 0.0295 58
+1712647972779 0.03 59
+1712647975702 0.3073 60
+1712647980013 0.032 61
+1712647982936 0.1144 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/total_flos

@@ -0,0 +1 @@
+1712647983084 5437004879757312.0 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/train_loss

@@ -0,0 +1 @@
+1712647983084 0.7901421915739775 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/train_runtime

@@ -0,0 +1 @@
+1712647983084 206.4235 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/train_samples_per_second

@@ -0,0 +1 @@
+1712647983084 2.403 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/metrics/train_steps_per_second

@@ -0,0 +1 @@
+1712647983084 0.3 62

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/_name_or_path

@@ -0,0 +1 @@
+mistralai/Mistral-7B-v0.1

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/accelerator_config

@@ -0,0 +1 @@
+{'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True}

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/adafactor

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/adam_beta1

@@ -0,0 +1 @@
+0.9

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/adam_beta2

@@ -0,0 +1 @@
+0.999

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/adam_epsilon

@@ -0,0 +1 @@
+1e-08

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/add_cross_attention

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/architectures

@@ -0,0 +1 @@
+['MistralForCausalLM']

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/attention_dropout

@@ -0,0 +1 @@
+0.0

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/auto_find_batch_size

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bad_words_ids

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/begin_suppress_tokens

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bench_dataset

@@ -0,0 +1 @@
+pharaouk/dharma-1/dharma_1_mini.json

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bench_source_max_len

@@ -0,0 +1 @@
+2048

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bench_split

@@ -0,0 +1 @@
+eval

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bf16

@@ -0,0 +1 @@
+True

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bf16_full_eval

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/bos_token_id

@@ -0,0 +1 @@
+1

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/chunk_size_feed_forward

@@ -0,0 +1 @@
+0

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/cosine_constant_lr_ratio

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/cosine_min_lr_ratio

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/cross_attention_hidden_size

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/data_seed

@@ -0,0 +1 @@
+None

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/dataloader_drop_last

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/dataloader_num_workers

@@ -0,0 +1 @@
+0

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/dataloader_persistent_workers

@@ -0,0 +1 @@
+False

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/dataloader_pin_memory

@@ -0,0 +1 @@
+True

mlruns/0/7e75ece8e18e485db64e4e2d9196e738/params/dataloader_prefetch_factor

@@ -0,0 +1 @@
+None