Upload 8 files

README.md

@@ -1,3 +1,19 @@
----
-license: apache-2.0
----
+# Makeshift Qwen2
+
+## Introduction
+Recently I've been interested in LLMs and wanted to train my own from scratch using the Qwen2 architecture provided through the Hugging Face transformers library. This was created locally on my personal laptop and is not powerful enough to be useful in any way, but it can respond to simple queries. I would recommend using a better-trained lightweight model instead of this one, as I've observed that although explicit in your queries, it often hallucinates data such as fictional U.S. Presidents or starts ranting about Chicago when told "Hey". The only advantage I can point out is its small size, weighing in at only 203 MB.
+
+## Model Details
+- **Model Name:** Makeshift LLM
+- **Architecture:** Qwen2-based
+- **Context:** 1024 Tokens
+- **Vocab Size:** 32,000 tokens
+- **Qwen2 Specific:** Hidden size of 768, 6 layers, 6 heads
+
+## Training Details
+- **GPU:** NVIDIA GeForce RTX 4070 Laptop GPU
+- **Cuda:** CUDA was used during pre-training and fine-tuning.
+- **VRAM:** 8gb
+
+A 28.4 GB subset of the AllenAI C4 English dataset was used for pre-training as well as for generating the tokenizer. However, the model was only trained up to an epoch of 0.77 (77% complete) because the loss was very stable at 3.5, and I didn't see any reason to continue training. Pre-training took about 18.5 hours with the GPU overclocked to its maximum capacity. Post-training involved 4 epochs of [databricks/databricks-dolly-15k](https://huggingface.co/datasets/databricks/databricks-dolly-15k) formatted in ChatML.
+

config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 1024,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 6,
+  "num_hidden_layers": 6,
+  "num_key_value_heads": 6,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "transformers_version": "4.55.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd5488022abd235caebef05ab39c621ed39e255c714df9c88c9a2c8522388c13
+size 211606032

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|unk|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% elif message['role'] == 'user' %}{{ '<|im_start|>user\n' + message['content'] + '<|im_end|>\n' }}{% elif message['role'] == 'assistant' %}{{ '<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|unk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|unk|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:408fed5aa1d887e2cf86c9eed36f0a99619e35f6477023fc07efb50e3fc02faa
+size 6097