Upload 3 files

finetune.py

@@ -0,0 +1,56 @@
+#!/home/dm/miniconda3/bin/python3
+from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
+from datasets import load_dataset
+
+# Load model and tokenizer
+model_name = "t5-small"  # or another transformer-based model
+tokenizer = T5Tokenizer.from_pretrained(model_name)
+model = T5ForConditionalGeneration.from_pretrained(model_name)
+
+# Load dataset
+dataset = load_dataset("json", data_files={"train": "train.json"})  
+evalset = load_dataset("json", data_files={"eval":  "eval.json"})
+
+def preprocess_function(examples):
+    inputs = ["Generate a question for: " + (ans if isinstance(ans, str) else "Unknown") for ans in examples["answer"]]
+    model_inputs = tokenizer(inputs, max_length=512, truncation=True, padding="max_length")  # <-- Added padding
+
+    labels = [q if isinstance(q, str) else "" for q in examples["question"]]
+    labels = tokenizer(labels, max_length=128, truncation=True, padding="max_length")  # <-- Added padding
+
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+tokenized_datasets = dataset.map(preprocess_function, batched=True)
+tokenized_evalsets = evalset.map(preprocess_function, batched=True)
+
+# Define training arguments
+training_args = TrainingArguments(
+    output_dir="./results",
+    evaluation_strategy="epoch",
+    save_strategy="epoch",
+    per_device_train_batch_size=8,
+    per_device_eval_batch_size=8,
+    num_train_epochs=3,
+    weight_decay=0.01,
+    logging_dir="./logs",
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_datasets["train"],
+    eval_dataset=tokenized_evalsets["eval"]
+)
+
+# Train model
+trainer.train()
+
+# Save trained model
+output_dir = "/home/dm/chat/AQ/aq_model"  # Change the folder name if needed
+trainer.save_model(output_dir)
+tokenizer.save_pretrained(output_dir)
+
+print(f"Model saved to {output_dir}")
+
+

split_train_eval.py

@@ -0,0 +1,15 @@
+#!/home/dm/miniconda3/bin/python3
+import json
+from sklearn.model_selection import train_test_split
+
+with open('inverted.json', 'r') as f:
+    data = json.load(f)
+    
+train_data, eval_data = train_test_split(data, test_size=0.2, random_state=42)
+
+with open('train.json', 'w') as f:
+    json.dump(train_data, f, indent=4)
+
+with open('eval.json', 'w') as f:
+    json.dump(eval_data, f, indent=4)
+

test_aqmodel.py

@@ -0,0 +1,24 @@
+#!/home/dm/miniconda3/bin/python3
+import sys
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+if len(sys.argv)<2:
+   print("Usage: python3 conversation.py '<your answer here>'")
+   sys.exit(1)
+
+# Define model path
+model_path = "/home/dm/chat/AQ/aq_model_b8"  # Make sure this points to your saved directory
+
+# Load model and tokenizer
+model = T5ForConditionalGeneration.from_pretrained(model_path)
+tokenizer = T5Tokenizer.from_pretrained(model_path)
+
+print("Model loaded successfully!")
+
+def generate_question(answer):
+    input_text = "Generate a question for: " + answer
+    input_ids = tokenizer(input_text, return_tensors="pt").input_ids
+    output_ids = model.generate(input_ids, max_length=50)
+    return tokenizer.decode(output_ids[0], skip_special_tokens=True)
+
+print(generate_question(sys.argv[1]))