Upload grpo_training.py with huggingface_hub

grpo_training.py

@@ -60,9 +60,27 @@ config = GRPOConfig(
     project="ligaments-sec-alignment",
 )
 
+# Define reward function for GRPO
+def preference_reward_func(samples):
+    """Simple reward function based on response length preference"""
+    rewards = []
+    for sample in samples:
+        # Prefer shorter, more concise responses (addressing verbosity issue)
+        response_length = len(sample["response"].split())
+        # Reward shorter responses (up to a reasonable length)
+        if response_length < 50:
+            reward = 1.0
+        elif response_length < 100:
+            reward = 0.5
+        else:
+            reward = 0.0  # Penalize overly verbose responses
+        rewards.append(reward)
+    return rewards
+
 # Initialize GRPO trainer
 trainer = GRPOTrainer(
     model=model_name,
+    reward_funcs=[preference_reward_func],
     train_dataset=train_dataset,
     eval_dataset=eval_dataset,
     peft_config=LoraConfig(