after transfer

.gitattributes

@@ -0,0 +1,10 @@
+*.png filter=xet merge=xet
+*.jpg filter=xet merge=xet
+*.jpeg filter=xet merge=xet
+*.gif filter=xet merge=xet
+*.pdf filter=xet merge=xet
+*.zip filter=xet merge=xet
+*.pth filter=xet merge=xet
+*.pt filter=xet merge=xet
+*.ckpt filter=xet merge=xet
+*.safetensors filter=xet merge=xet

scripts/train_unsloth.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 """
 GridMind-RL Unsloth GRPO Training Script
-----------------------------------------
-Fine-tunes Qwen2.5-0.5B-Instruct using Unsloth's 4-bit LoRA and TRL's GRPOTrainer.
+----------------------------------------------
+Fine-tunes Qwen2.5-1.5B-Instruct using Unsloth's 4-bit LoRA and TRL's GRPOTrainer.
 The environment rewards are gathered by hitting the OpenEnv HTTP server directly.
+
+FIXED: Removed reward hacking, added entropy bonus, diverse seeds, proper normalization.
 """
 
 import argparse
@@ -13,16 +15,16 @@ import re
 import sys
 import requests
 import pandas as pd
+import random
+from collections import Counter
 from datasets import Dataset
 from trl import GRPOTrainer, GRPOConfig
 from unsloth import FastLanguageModel
 from transformers import TrainerCallback
 
-# Ensure results directory exists
 os.makedirs("results", exist_ok=True)
 
-SYSTEM_PROMPT = """\
-You are an expert industrial building energy controller.
+SYSTEM_PROMPT = """You are an expert industrial building energy controller.
 Each turn you receive the current building state and must respond with 
 ONLY a valid JSON action object.
 
@@ -31,24 +33,24 @@ Action format:
  "batch_job_slot": <0-4>, "load_shed_fraction": <0.0-0.5>, "building_id": 0}
 
 Strategy:
-- Charge storage when price < $0.08/kWh (positive thermal_charge_rate)
-- Discharge storage when price > $0.15/kWh (negative thermal_charge_rate)  
-- Shed load 0.3-0.5 when grid_stress_signal > 0.7
-- Reduce HVAC during peak hours (8-12, 17-21)
-- Keep temperature between 19-23°C"""
+- Always respond with valid JSON containing all required keys
+- Vary your actions - don't repeat the same pattern
+- Optimize for low cost + comfort maintenance + grid response"""
 
-def make_prompt(i):
+def make_prompt(i, obs=None, task_desc=""):
+    system_content = SYSTEM_PROMPT
+    if obs and task_desc:
+        system_content += f"\n\nCurrent observation:\n- Temperature: {obs.get('indoor_temperature', 21):.1f}°C\n- Price: ${obs.get('current_price', 0.10):.3f}/kWh\n- Grid stress: {obs.get('grid_stress_signal', 0):.2f}\n- Hour: {obs.get('hour_of_day', 12)}\n- Storage: {obs.get('thermal_storage_level', 0.5):.1%}"
+    
     return [{
-        "role": "system", "content": SYSTEM_PROMPT
+        "role": "system", "content": system_content
     }, {
         "role": "user",
-        "content": f"Episode {i+1}: The building simulation is starting. "
-                   "You will receive the state each step. "
-                   "Output your first action as JSON now."
+        "content": f"Episode {i+1}: {task_desc}\nOutput action as JSON."
     }]
 
 def reward_valid_json(completions, **kwargs):
-    """Reward 0.3 for any valid JSON output."""
+    """Reward 0.25 for any valid JSON output."""
     rewards = []
     for completion in completions:
         text = completion[0]["content"] if isinstance(completion, list) else completion
@@ -56,7 +58,7 @@ def reward_valid_json(completions, **kwargs):
             match = re.search(r'\{.*?\}', text, re.DOTALL)
             if match:
                 json.loads(match.group())
-                rewards.append(0.3)
+                rewards.append(0.25)
             else:
                 rewards.append(0.0)
         except Exception:
@@ -64,7 +66,7 @@ def reward_valid_json(completions, **kwargs):
     return rewards
 
 def reward_has_required_keys(completions, **kwargs):
-    """Reward 0.3 if JSON has all 4 required action keys."""
+    """Reward 0.25 if JSON has all 4 required action keys."""
     required = {"hvac_power_level", "thermal_charge_rate", "batch_job_slot", "load_shed_fraction"}
     rewards = []
     for completion in completions:
@@ -74,7 +76,7 @@ def reward_has_required_keys(completions, **kwargs):
             if match:
                 action = json.loads(match.group())
                 if required.issubset(action.keys()):
-                    rewards.append(0.3)
+                    rewards.append(0.25)
                 else:
                     rewards.append(0.1)
             else:
@@ -84,14 +86,17 @@ def reward_has_required_keys(completions, **kwargs):
     return rewards
 
 def get_reward_env_interaction(env_url):
-    """Episode-level reward from /grade endpoint with seed variation.
-
-    Uses 8-step rollouts with varied seeds to prevent mode collapse.
-    The /grade endpoint returns the true episode score (0.0-1.0 clamped),
-    which we use directly as the primary learning signal.
+    """Episode-level reward from /grade endpoint with diverse seeds.
+    
+    FIXED: Uses raw /grade score directly (0.0-1.0), no normalization that causes reward hacking.
+    Each sample gets a different seed/task to prevent mode collapse.
     """
+    last_observations = []
+    
     def reward_env_interaction(completions, **kwargs):
+        nonlocal last_observations
         rewards = []
+        
         for i, completion in enumerate(completions):
             text = completion[0]["content"] if isinstance(completion, list) else completion
             try:
@@ -105,9 +110,9 @@ def get_reward_env_interaction(env_url):
                     "building_id": 0
                 }
 
-                # Vary seed to prevent mode collapse — each rollout sees a different episode
-                seed = 1000 + i
-                task_id = (i % 3) + 1  # Cycle through tasks 1, 2, 3
+                # Diverse seeds to prevent mode collapse
+                seed = 2000 + (i * 17) % 500
+                task_id = (i % 3) + 1
 
                 reset_resp = requests.post(
                     f"{env_url}/reset",
@@ -118,7 +123,11 @@ def get_reward_env_interaction(env_url):
                     rewards.append(0.0)
                     continue
 
-                for _ in range(8):
+                obs = reset_resp.json().get("observations", [{}])[0] if reset_resp.json().get("observations") else {}
+                last_observations.append(obs)
+
+                # 4-step mini-rollout for faster training
+                for _ in range(4):
                     step_resp = requests.post(
                         f"{env_url}/step",
                         json=[step_action],
@@ -130,10 +139,7 @@ def get_reward_env_interaction(env_url):
                 grade_resp = requests.get(f"{env_url}/grade", timeout=30)
                 if grade_resp.status_code == 200:
                     episode_score = float(grade_resp.json().get("score", 0.5))
-                    # Normalize: heuristic baseline ≈ 0.5, zero-shot ≈ 0.65, trained ≈ 0.72
-                    # Map to 0.0-1.0 where 0.5 is the floor (heuristic), 0.72 is the ceiling (trained target)
-                    normalized = (episode_score - 0.4) / 0.32  # maps 0.4→0.0, 0.72→1.0
-                    rewards.append(max(0.0, min(1.0, normalized)))
+                    rewards.append(episode_score)
                 else:
                     rewards.append(0.0)
 
@@ -143,6 +149,30 @@ def get_reward_env_interaction(env_url):
         return rewards
     return reward_env_interaction
 
+def reward_entropy_bonus(completions, **kwargs):
+    """Reward action diversity to prevent mode collapse - bonus for varied actions."""
+    rewards = []
+    actions_seen = []
+    
+    for completion in completions:
+        text = completion[0]["content"] if isinstance(completion, list) else completion
+        try:
+            match = re.search(r'\{.*?\}', text, re.DOTALL)
+            if match:
+                action = json.loads(match.group())
+                actions_seen.append(json.dumps(action, sort_keys=True))
+        except:
+            pass
+    
+    if len(actions_seen) > 1:
+        unique_actions = len(set(actions_seen))
+        diversity_ratio = unique_actions / len(actions_seen)
+        rewards = [0.1 * diversity_ratio] * len(actions_seen)
+    else:
+        rewards = [0.05] * len(completions)
+    
+    return rewards
+
 class CSVLogCallback(TrainerCallback):
     """Custom callback to continuously log training metrics to a CSV file."""
     def __init__(self, output_path):
@@ -159,7 +189,7 @@ class CSVLogCallback(TrainerCallback):
 def main():
     parser = argparse.ArgumentParser(description="Train GridMind-RL agent with Unsloth GRPO")
     parser.add_argument("--env-url", type=str, default="http://localhost:7860", help="OpenEnv server URL")
-    parser.add_argument("--model-name", type=str, default="unsloth/Qwen2.5-0.5B-Instruct", help="Base model")
+    parser.add_argument("--model-name", type=str, default="unsloth/Qwen2.5-1.5B-Instruct", help="Base model")
     parser.add_argument("--prompts", type=int, default=300, help="Number of training prompts")
     parser.add_argument("--epochs", type=int, default=1, help="Training epochs")
     parser.add_argument("--max-steps", type=int, default=-1, help="Max steps (overrides epochs if > 0)")
@@ -169,7 +199,7 @@ def main():
 
     print(f"🚀 Loading model: {args.model_name}")
     max_seq_length = 512
-    lora_rank = 8
+    lora_rank = 16  # Increased for better learning capacity
 
     model, tokenizer = FastLanguageModel.from_pretrained(
         model_name=args.model_name,
@@ -199,16 +229,16 @@ def main():
         max_steps=args.max_steps,
         per_device_train_batch_size=1,
         gradient_accumulation_steps=4,
-        num_generations=4,  # GRPO group size
+        num_generations=4,
         max_prompt_length=256,
         max_completion_length=128,
-        learning_rate=5e-6,
+        learning_rate=3e-6,  # Lower LR for stability
         lr_scheduler_type="cosine",
         warmup_ratio=0.1,
         logging_steps=5,
         save_steps=100,
         fp16=True,
-        report_to="none",  # We use our CSV callback instead
+        report_to="none",
         seed=42,
     )
 
@@ -221,6 +251,7 @@ def main():
             reward_valid_json,
             reward_has_required_keys,
             get_reward_env_interaction(args.env_url),
+            reward_entropy_bonus,
         ],
         callbacks=[CSVLogCallback(args.output_csv)]
     )