Training in progress, step 5958

README.md

@@ -0,0 +1,53 @@
+---
+license: mit
+base_model: Josephgflowers/TinyLlama-3T-Cinder-v1.3
+tags:
+- generated_from_trainer
+model-index:
+- name: TinyLlama-Cinder-Agent-Rag
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# TinyLlama-Cinder-Agent-Rag
+
+This model is a fine-tuned version of [Josephgflowers/TinyLlama-3T-Cinder-v1.3](https://huggingface.co/Josephgflowers/TinyLlama-3T-Cinder-v1.3) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 12
+- eval_batch_size: 32
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 1.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.0.dev0
+- Pytorch 2.2.2+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.08850744042455e+17,
+    "train_loss": 0.8458019172076853,
+    "train_runtime": 81918.9773,
+    "train_samples": 71495,
+    "train_samples_per_second": 0.873,
+    "train_steps_per_second": 0.073
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Josephgflowers/TinyLlama-3T-Cinder-v1.2",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0543fbabf0edae6ff52542b33203d63e0b477429f6973399eef6468b412371a4
+size 4400216536

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 9.08850744042455e+17,
+    "train_loss": 0.8458019172076853,
+    "train_runtime": 81918.9773,
+    "train_samples": 71495,
+    "train_samples_per_second": 0.873,
+    "train_steps_per_second": 0.073
+}

trainer_state.json

@@ -0,0 +1,455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 5958,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.016784155756965426,
+      "grad_norm": 90801.3515625,
+      "learning_rate": 4.916079221215173e-05,
+      "loss": 0.9925,
+      "step": 100
+    },
+    {
+      "epoch": 0.03356831151393085,
+      "grad_norm": 83764.0078125,
+      "learning_rate": 4.832158442430346e-05,
+      "loss": 0.9625,
+      "step": 200
+    },
+    {
+      "epoch": 0.050352467270896276,
+      "grad_norm": 84057.3125,
+      "learning_rate": 4.748237663645519e-05,
+      "loss": 0.934,
+      "step": 300
+    },
+    {
+      "epoch": 0.0671366230278617,
+      "grad_norm": 76437.0546875,
+      "learning_rate": 4.664316884860692e-05,
+      "loss": 0.9354,
+      "step": 400
+    },
+    {
+      "epoch": 0.08392077878482712,
+      "grad_norm": 76537.3359375,
+      "learning_rate": 4.5803961060758646e-05,
+      "loss": 0.934,
+      "step": 500
+    },
+    {
+      "epoch": 0.10070493454179255,
+      "grad_norm": 86227.09375,
+      "learning_rate": 4.4964753272910375e-05,
+      "loss": 0.9027,
+      "step": 600
+    },
+    {
+      "epoch": 0.11748909029875797,
+      "grad_norm": 86583.5078125,
+      "learning_rate": 4.4125545485062104e-05,
+      "loss": 0.8981,
+      "step": 700
+    },
+    {
+      "epoch": 0.1342732460557234,
+      "grad_norm": 59143.1171875,
+      "learning_rate": 4.328633769721383e-05,
+      "loss": 0.9329,
+      "step": 800
+    },
+    {
+      "epoch": 0.1510574018126888,
+      "grad_norm": 77289.375,
+      "learning_rate": 4.244712990936556e-05,
+      "loss": 0.9125,
+      "step": 900
+    },
+    {
+      "epoch": 0.16784155756965424,
+      "grad_norm": 66578.9765625,
+      "learning_rate": 4.160792212151729e-05,
+      "loss": 0.9198,
+      "step": 1000
+    },
+    {
+      "epoch": 0.18462571332661967,
+      "grad_norm": 64711.90234375,
+      "learning_rate": 4.076871433366902e-05,
+      "loss": 0.8985,
+      "step": 1100
+    },
+    {
+      "epoch": 0.2014098690835851,
+      "grad_norm": 74255.734375,
+      "learning_rate": 3.992950654582075e-05,
+      "loss": 0.9154,
+      "step": 1200
+    },
+    {
+      "epoch": 0.21819402484055053,
+      "grad_norm": 76627.859375,
+      "learning_rate": 3.9090298757972476e-05,
+      "loss": 0.907,
+      "step": 1300
+    },
+    {
+      "epoch": 0.23497818059751593,
+      "grad_norm": 69068.90625,
+      "learning_rate": 3.8251090970124205e-05,
+      "loss": 0.911,
+      "step": 1400
+    },
+    {
+      "epoch": 0.25176233635448136,
+      "grad_norm": 63122.66796875,
+      "learning_rate": 3.7411883182275934e-05,
+      "loss": 0.8597,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2685464921114468,
+      "grad_norm": 76006.2265625,
+      "learning_rate": 3.657267539442766e-05,
+      "loss": 0.884,
+      "step": 1600
+    },
+    {
+      "epoch": 0.2853306478684122,
+      "grad_norm": 64542.19140625,
+      "learning_rate": 3.573346760657939e-05,
+      "loss": 0.8884,
+      "step": 1700
+    },
+    {
+      "epoch": 0.3021148036253776,
+      "grad_norm": 65309.5546875,
+      "learning_rate": 3.489425981873112e-05,
+      "loss": 0.8707,
+      "step": 1800
+    },
+    {
+      "epoch": 0.3188989593823431,
+      "grad_norm": 81841.5546875,
+      "learning_rate": 3.405505203088285e-05,
+      "loss": 0.8818,
+      "step": 1900
+    },
+    {
+      "epoch": 0.3356831151393085,
+      "grad_norm": 74731.21875,
+      "learning_rate": 3.321584424303458e-05,
+      "loss": 0.8635,
+      "step": 2000
+    },
+    {
+      "epoch": 0.35246727089627394,
+      "grad_norm": 59748.52734375,
+      "learning_rate": 3.2376636455186307e-05,
+      "loss": 0.8827,
+      "step": 2100
+    },
+    {
+      "epoch": 0.36925142665323935,
+      "grad_norm": 78755.6796875,
+      "learning_rate": 3.1537428667338035e-05,
+      "loss": 0.8564,
+      "step": 2200
+    },
+    {
+      "epoch": 0.38603558241020475,
+      "grad_norm": 68127.890625,
+      "learning_rate": 3.0698220879489764e-05,
+      "loss": 0.8619,
+      "step": 2300
+    },
+    {
+      "epoch": 0.4028197381671702,
+      "grad_norm": 80441.515625,
+      "learning_rate": 2.9859013091641493e-05,
+      "loss": 0.8421,
+      "step": 2400
+    },
+    {
+      "epoch": 0.4196038939241356,
+      "grad_norm": 69039.3203125,
+      "learning_rate": 2.9019805303793218e-05,
+      "loss": 0.8639,
+      "step": 2500
+    },
+    {
+      "epoch": 0.43638804968110106,
+      "grad_norm": 70787.9921875,
+      "learning_rate": 2.818059751594495e-05,
+      "loss": 0.8428,
+      "step": 2600
+    },
+    {
+      "epoch": 0.45317220543806647,
+      "grad_norm": 73120.625,
+      "learning_rate": 2.734138972809668e-05,
+      "loss": 0.8595,
+      "step": 2700
+    },
+    {
+      "epoch": 0.46995636119503187,
+      "grad_norm": 63243.34375,
+      "learning_rate": 2.6502181940248405e-05,
+      "loss": 0.8359,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4867405169519973,
+      "grad_norm": 69866.1015625,
+      "learning_rate": 2.5662974152400137e-05,
+      "loss": 0.8351,
+      "step": 2900
+    },
+    {
+      "epoch": 0.5035246727089627,
+      "grad_norm": 70137.2109375,
+      "learning_rate": 2.4823766364551865e-05,
+      "loss": 0.8191,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5203088284659282,
+      "grad_norm": 75627.3203125,
+      "learning_rate": 2.398455857670359e-05,
+      "loss": 0.8473,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5370929842228936,
+      "grad_norm": 66008.0,
+      "learning_rate": 2.3145350788855323e-05,
+      "loss": 0.8263,
+      "step": 3200
+    },
+    {
+      "epoch": 0.553877139979859,
+      "grad_norm": 65289.640625,
+      "learning_rate": 2.2306143001007052e-05,
+      "loss": 0.8386,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5706612957368244,
+      "grad_norm": 73041.9921875,
+      "learning_rate": 2.1466935213158777e-05,
+      "loss": 0.8287,
+      "step": 3400
+    },
+    {
+      "epoch": 0.5874454514937899,
+      "grad_norm": 66999.875,
+      "learning_rate": 2.062772742531051e-05,
+      "loss": 0.8203,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6042296072507553,
+      "grad_norm": 68238.859375,
+      "learning_rate": 1.9788519637462235e-05,
+      "loss": 0.831,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6210137630077207,
+      "grad_norm": 75512.6796875,
+      "learning_rate": 1.8949311849613967e-05,
+      "loss": 0.8046,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6377979187646862,
+      "grad_norm": 69572.578125,
+      "learning_rate": 1.8110104061765696e-05,
+      "loss": 0.8075,
+      "step": 3800
+    },
+    {
+      "epoch": 0.6545820745216515,
+      "grad_norm": 70714.1953125,
+      "learning_rate": 1.727089627391742e-05,
+      "loss": 0.8386,
+      "step": 3900
+    },
+    {
+      "epoch": 0.671366230278617,
+      "grad_norm": 70761.921875,
+      "learning_rate": 1.6431688486069153e-05,
+      "loss": 0.7866,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6881503860355824,
+      "grad_norm": 69547.9375,
+      "learning_rate": 1.559248069822088e-05,
+      "loss": 0.8212,
+      "step": 4100
+    },
+    {
+      "epoch": 0.7049345417925479,
+      "grad_norm": 69571.3046875,
+      "learning_rate": 1.4753272910372609e-05,
+      "loss": 0.7977,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7217186975495132,
+      "grad_norm": 64268.54296875,
+      "learning_rate": 1.391406512252434e-05,
+      "loss": 0.7886,
+      "step": 4300
+    },
+    {
+      "epoch": 0.7385028533064787,
+      "grad_norm": 70546.71875,
+      "learning_rate": 1.3074857334676067e-05,
+      "loss": 0.7956,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7552870090634441,
+      "grad_norm": 67713.59375,
+      "learning_rate": 1.2235649546827795e-05,
+      "loss": 0.7939,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7720711648204095,
+      "grad_norm": 69085.0703125,
+      "learning_rate": 1.1396441758979524e-05,
+      "loss": 0.7857,
+      "step": 4600
+    },
+    {
+      "epoch": 0.788855320577375,
+      "grad_norm": 76299.1015625,
+      "learning_rate": 1.0557233971131253e-05,
+      "loss": 0.7889,
+      "step": 4700
+    },
+    {
+      "epoch": 0.8056394763343404,
+      "grad_norm": 84437.421875,
+      "learning_rate": 9.718026183282982e-06,
+      "loss": 0.7926,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8224236320913058,
+      "grad_norm": 65666.4375,
+      "learning_rate": 8.87881839543471e-06,
+      "loss": 0.7792,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8392077878482712,
+      "grad_norm": 66860.4765625,
+      "learning_rate": 8.039610607586439e-06,
+      "loss": 0.764,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8559919436052367,
+      "grad_norm": 75992.640625,
+      "learning_rate": 7.200402819738168e-06,
+      "loss": 0.7925,
+      "step": 5100
+    },
+    {
+      "epoch": 0.8727760993622021,
+      "grad_norm": 70527.9765625,
+      "learning_rate": 6.361195031889897e-06,
+      "loss": 0.7864,
+      "step": 5200
+    },
+    {
+      "epoch": 0.8895602551191675,
+      "grad_norm": 74602.359375,
+      "learning_rate": 5.5219872440416254e-06,
+      "loss": 0.7991,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9063444108761329,
+      "grad_norm": 64026.41796875,
+      "learning_rate": 4.682779456193353e-06,
+      "loss": 0.7762,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9231285666330984,
+      "grad_norm": 62320.39453125,
+      "learning_rate": 3.843571668345083e-06,
+      "loss": 0.789,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9399127223900637,
+      "grad_norm": 59294.8984375,
+      "learning_rate": 3.0043638804968113e-06,
+      "loss": 0.7888,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9566968781470292,
+      "grad_norm": 67846.1953125,
+      "learning_rate": 2.16515609264854e-06,
+      "loss": 0.7874,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9734810339039947,
+      "grad_norm": 67274.59375,
+      "learning_rate": 1.3259483048002687e-06,
+      "loss": 0.7928,
+      "step": 5800
+    },
+    {
+      "epoch": 0.99026518966096,
+      "grad_norm": 72139.2734375,
+      "learning_rate": 4.867405169519974e-07,
+      "loss": 0.7832,
+      "step": 5900
+    },
+    {
+      "epoch": 1.0,
+      "step": 5958,
+      "total_flos": 9.08850744042455e+17,
+      "train_loss": 0.8458019172076853,
+      "train_runtime": 81918.9773,
+      "train_samples_per_second": 0.873,
+      "train_steps_per_second": 0.073
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 5958,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5958,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.08850744042455e+17,
+  "train_batch_size": 12,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbd6ecac03d543acbcd036abe674eba4f3d9c6b10013486b12a0549c7e08434e
+size 5112