Uploading the model

FineTuning_Cyber_LLM.py

@@ -0,0 +1,58 @@
+import torch
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config, TextDataset, DataCollatorForLanguageModeling, Trainer, TrainingArguments
+import os
+
+# Define your book data file
+book_data_file = "data\Computer Networking_cleaned.txt"
+
+# Load the book data
+with open(book_data_file, "r", encoding="utf-8") as f:
+    text = f.read()
+
+# Initialize a GPT-2 model and tokenizer
+model_name = "gpt2"  # You can choose a different model size as needed
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+config = GPT2Config.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name, config=config)
+
+# Split the text into smaller chunks
+max_sequence_length = 1024
+chunks = [text[i:i+max_sequence_length] for i in range(0, len(text), max_sequence_length)]
+
+# Initialize an empty list for input_ids
+input_ids = []
+
+# Tokenize the text data
+for chunk in chunks:
+    input_ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
+
+# Create a dataset and data collator for language modeling
+dataset = TextDataset(tokenizer=tokenizer, file_path=book_data_file, block_size=128, overwrite_cache=False)
+
+# Set up training arguments
+training_args = TrainingArguments(
+    output_dir="./Cyber_LLM",
+    overwrite_output_dir=True,
+    num_train_epochs=1,  # You can adjust the number of training epochs
+    per_device_train_batch_size=32,
+    save_steps=10_000,
+    save_total_limit=2,
+    evaluation_strategy="steps",
+    eval_steps=10_000,
+)
+
+# Initialize a trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False),
+    train_dataset=dataset,
+)
+
+# Train the model
+trainer.train()
+
+# Save the model
+trainer.save_model("./Cyber_LLM")
+
+print("Training completed.")

FineTuning_Cyber_LLM_v2.py

@@ -0,0 +1,80 @@
+import os
+import torch
+from torch.utils.data import Dataset
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config, DataCollatorForLanguageModeling, Trainer, TrainingArguments
+
+class CustomTextDataset(Dataset):
+    def __init__(self, tokenizer, data_chunk, block_size):
+        self.examples = []
+        for chunk in data_chunk:
+            tokenized_text = tokenizer.encode(chunk, add_special_tokens=True)
+            self.examples.extend(tokenized_text)
+
+        self.block_size = block_size
+
+    def __len__(self):
+        return len(self.examples) - self.block_size
+
+    def __getitem__(self, i):
+        # Return a chunk of length block_size
+        return torch.tensor(self.examples[i:i + self.block_size])
+
+# Define the folder containing text files
+folder_path = "data"
+
+# List all files in the folder
+file_list = [f for f in os.listdir(folder_path) if f.endswith(".txt")]
+
+# Initialize an empty list to store all text data
+all_text_data = []
+
+# Read all files in the folder and concatenate their contents
+for file_name in file_list:
+    file_path = os.path.join(folder_path, file_name)
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_text = f.read()
+        all_text_data.append(file_text)
+
+# Concatenate all text data
+text = " ".join(all_text_data)
+
+# Initialize a GPT-2 model and tokenizer
+model_name = "gpt2"  # You can choose a different model size as needed
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+config = GPT2Config.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name, config=config)
+
+# Split the text into smaller chunks
+max_sequence_length = 1024
+chunks = [text[i:i + max_sequence_length] for i in range(0, len(text), max_sequence_length)]
+
+# Initialize a custom dataset
+dataset = CustomTextDataset(tokenizer=tokenizer, data_chunk=chunks, block_size=128)
+
+# Set up training arguments
+training_args = TrainingArguments(
+    output_dir="./Cyber_LLM",
+    overwrite_output_dir=True,
+    num_train_epochs=1,  # You can adjust the number of training epochs
+    per_device_train_batch_size=32,
+    save_steps=10_000,
+    save_total_limit=2,
+    evaluation_strategy="epoch",  # Adjusted to "epoch"
+    eval_steps=10_000,
+)
+
+# Initialize a trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False),
+    train_dataset=dataset,
+)
+
+# Train the model
+trainer.train()
+
+# Save the model
+model.save_pretrained("./Cyber_LLM")
+
+print("Training completed.")

FineTuning_Cyber_LLM_v3.py

@@ -0,0 +1,63 @@
+import os
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config, TextDataset, DataCollatorForLanguageModeling, Trainer, TrainingArguments
+
+# Define your folder containing data files
+data_folder = "data"
+
+# Initialize a GPT-2 model and tokenizer
+model_name = "gpt2"  # You can choose a different model size as needed
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+config = GPT2Config.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name, config=config)
+
+# Initialize an empty list for input_ids
+input_ids = []
+
+# Read and process each file in the folder
+for filename in os.listdir(data_folder):
+    file_path = os.path.join(data_folder, filename)
+
+    # Check if the path is a file
+    if os.path.isfile(file_path):
+        # Load the file data
+        with open(file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+
+        # Split the text into smaller chunks
+        max_sequence_length = 1024
+        chunks = [text[i:i+max_sequence_length] for i in range(0, len(text), max_sequence_length)]
+
+        # Tokenize the text data
+        for chunk in chunks:
+            input_ids.extend(tokenizer.encode(chunk, add_special_tokens=True))
+
+# Create a dataset and data collator for language modeling
+dataset = TextDataset(tokenizer=tokenizer, inputs=input_ids, block_size=128)
+
+# Set up training arguments
+training_args = TrainingArguments(
+    output_dir="./Cyber_LLM",
+    overwrite_output_dir=True,
+    num_train_epochs=3,  # You can adjust the number of training epochs
+    per_device_train_batch_size=4,  # Adjust based on your GPU memory
+    save_steps=10_000,
+    save_total_limit=2,
+    evaluation_strategy="epoch",
+    eval_steps=10_000,
+)
+
+# Initialize a trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False),
+    train_dataset=dataset,
+)
+
+# Train the model
+trainer.train()
+
+# Save the model
+model.save_pretrained("./Cyber_LLM")
+
+print("Training completed.")