Phase 2 complete: Fixed inference loop and added phase gates

apps/audit_api.py

@@ -0,0 +1,23 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+import uvicorn
+
+app = FastAPI(title="Compliance Audit API")
+logs = []
+
+class AuditRecord(BaseModel):
+    ad_id: str
+    action_taken: str
+    reasoning: str
+
+@app.post("/log")
+def log_audit(record: AuditRecord):
+    logs.append(record.dict())
+    return {"status": "success", "audit_id": f"AUD-{len(logs)}"}
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "service": "compliance-audit"}
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8003)

apps/crm_api.py

@@ -0,0 +1,20 @@
+from fastapi import FastAPI
+import uvicorn
+
+app = FastAPI(title="Advertiser CRM API")
+
+ADVERTISERS = {
+    "adv_001": {"prior_violations": 3, "summary": "High-risk repeat offender."},
+    "adv_002": {"prior_violations": 0, "summary": "Established clean record."}
+}
+
+@app.get("/advertiser/{advertiser_id}")
+def get_advertiser(advertiser_id: str):
+    return ADVERTISERS.get(advertiser_id, {"prior_violations": 0, "summary": "New advertiser."})
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "service": "advertiser-crm"}
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8002)

apps/regulatory_api.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+import uvicorn
+
+app = FastAPI(title="Regulatory DB API")
+
+REGULATIONS = {
+    "healthcare": {
+        "policy_summary": "Claims require FDA approval. No 'guaranteed results' allowed.",
+        "risk_level": "high"
+    },
+    "financial": {
+        "policy_summary": "Requires SEC registration. Prohibited: predatory APR > 36%.",
+        "risk_level": "high"
+    },
+    "general": {
+        "policy_summary": "Standard standards apply. No deceptive claims.",
+        "risk_level": "low"
+    }
+}
+
+@app.get("/regulations/{category}")
+def get_regulations(category: str):
+    return REGULATIONS.get(category.lower(), REGULATIONS["general"])
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "service": "regulatory-db"}
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8001)

apps/start_all.bat

@@ -0,0 +1,10 @@
+@echo off
+echo Launching Enterprise Ecosystem...
+
+:: This line forces Windows to go to the project root before running anything
+cd /d "%~dp0\.."
+
+start "Regulatory API" cmd /k "uv run python apps\regulatory_api.py"
+start "CRM API" cmd /k "uv run python apps\crm_api.py"
+start "Audit API" cmd /k "uv run python apps\audit_api.py"
+start "Environment Server" cmd /k "uv run uvicorn server.app:app --host 0.0.0.0 --port 8000"

grpo_train.py

@@ -0,0 +1,56 @@
+import torch
+from unsloth import FastLanguageModel, PatchFastRL
+from trl import GRPOTrainer, GRPOConfig
+from src.environment import AdPolicyEnvironment
+
+# 1. Load Model with Unsloth
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "unsloth/Llama-3.1-8B-Instruct",
+    max_seq_length = 1024,
+    load_in_4bit = True,
+)
+
+# 2. Define Reward Functions
+def reward_compliance(prompts, completions, **kwargs):
+    rewards = []
+    for completion in completions:
+        # Check if the model called the necessary tools in order
+        if "query_regulations" in completion and "submit_audit" in completion:
+            rewards.append(2.0)
+        else:
+            rewards.append(0.0)
+    return rewards
+
+def reward_json_format(prompts, completions, **kwargs):
+    rewards = []
+    for completion in completions:
+        try:
+            import json
+            json.loads(completion)
+            rewards.append(1.0)
+        except:
+            rewards.append(0.0)
+    return rewards
+
+# 3. Configure Trainer
+training_args = GRPOConfig(
+    output_dir = "outputs/meta-ad-agent",
+    learning_rate = 5e-6,
+    num_train_epochs = 1,
+    per_device_train_batch_size = 4,
+    gradient_accumulation_steps = 4,
+    max_prompt_length = 512,
+    max_completion_length = 512,
+    num_generations = 8, # Number of variations to compare
+)
+
+trainer = GRPOTrainer(
+    model = model,
+    reward_funcs = [reward_compliance, reward_json_format],
+    args = training_args,
+    train_dataset = [], # We will stream data from your AdGenerator here
+    tokenizer = tokenizer,
+)
+
+# 4. Start Training
+# trainer.train()

inference.py

@@ -6,7 +6,7 @@ from openai import OpenAI
 # 1. MANDATORY VARIABLES EXACTLY AS REQUESTED BY SCALAR
 API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
 API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY", "dummy_local_token")
-MODEL_NAME = os.getenv("MODEL_NAME", "meta-llama/Llama-3-70b-chat-hf")
+MODEL_NAME = os.getenv("MODEL_NAME", "meta-llama/Meta-Llama-3-8B-Instruct")
 
 ENV_URL = "http://localhost:8000"
 MAX_STEPS = 10
@@ -39,17 +39,28 @@ def log_end(success: bool, steps: int, score: float, rewards: list) -> None:
 
 def get_llm_action(observation_data):
     """Asks the LLM what action to take based on the ad observation."""
-    system_prompt = """You are an expert Meta Ad-Policy Moderator AI. 
-    Evaluate the ad and output a decision. Using tools costs -0.05 points, so be efficient.
-    
+    system_prompt = """You are an enterprise Ad Policy Compliance Agent.
+    You navigate a multi-system compliance workflow. Always respond with ONLY valid JSON.
+
+    REQUIRED PHASE ORDER:
+    1. query_regulations   — always first
+    2. analyze_image       — required for visual/multimodal tasks
+    3. check_advertiser_history or request_landing_page — as needed
+    4. submit_audit        — always before final decision
+    5. approve or reject   — final decision only after audit
+
     AVAILABLE ACTIONS:
+    - query_regulations
     - analyze_image
+    - check_advertiser_history
     - request_landing_page
     - request_id_verification
+    - submit_audit
     - approve
     - reject
-    
-    You MUST respond in valid JSON format containing "action_type" and "reasoning".
+
+    Response format:
+    {"action_type": "<action>", "reasoning": "<brief reason>"}
     """
 
     user_prompt = f"Current Ad Observation:\n{json.dumps(observation_data, indent=2)}\n\nWhat is your next action?"
@@ -61,17 +72,25 @@ def get_llm_action(observation_data):
                 {"role": "system", "content": system_prompt},
                 {"role": "user", "content": user_prompt}
             ],
-            response_format={"type": "json_object"},
+            # Removed response_format={"type": "json_object"} as HF router often rejects it
             temperature=0.1
         )
         
-        result = json.loads(response.choices[0].message.content)
+        # Clean the response in case the LLM wrapped it in markdown code blocks like ```json ... ```
+        content = response.choices[0].message.content.strip()
+        if content.startswith("```json"):
+            content = content[7:-3].strip()
+        elif content.startswith("```"):
+            content = content[3:-3].strip()
+            
+        result = json.loads(content)
         return {
-            "action_type": result.get("action_type", "analyze_image"),
+            "action_type": result.get("action_type", "query_regulations"),
             "reasoning": result.get("reasoning", "Fallback reasoning")
         }
     except Exception as e:
-        return {"action_type": "analyze_image", "reasoning": "Error recovery."}
+        print(f"\n[CRITICAL LLM ERROR]: {str(e)}\n", flush=True) # THIS WILL REVEAL THE BUG
+        return {"action_type": "query_regulations", "reasoning": f"Error recovery: {str(e)}"}
 
 def main() -> None:
     for task_id in TASKS:
@@ -82,29 +101,42 @@ def main() -> None:
         success = False
         
         try:
+            # 1. Reset the environment
             res = requests.post(f"{ENV_URL}/reset", json={"task_id": task_id})
             if res.status_code != 200:
                 log_step(step=1, action="reset_failed", reward=0.0, done=True, error=f"HTTP {res.status_code}")
-                # Forced score to 0.01 instead of 0.0
                 log_end(success=False, steps=0, score=0.01, rewards=[])
                 continue
                 
-            data = res.json()
-            observation = data.get("observation", data)
+            # 2. Initialize data from the reset
+            step_data = res.json() 
+            observation = step_data.get("observation", step_data)
             done = False
             
+            # 3. THE SINGLE LOOP (Fixed)
             while not done and steps_taken < MAX_STEPS:
                 steps_taken += 1
                 
+                # Feedback memory for the LLM
+                llm_observation = {
+                    "task_id": task_id,
+                    "last_feedback": step_data.get("status_message", "No feedback yet."),
+                    "step_count": steps_taken,
+                    "ad_details": observation 
+                }
+                
                 # Get action from LLM
-                action_payload = get_llm_action(observation)
+                action_payload = get_llm_action(llm_observation)
                 action_str = action_payload["action_type"]
+                if "Error code: 402" in action_payload.get("reasoning", ""):
+                 done = True
+                 log_step(step=steps_taken, action=action_str, reward=0.0, done=True, error="API credits depleted")
+                 break
+                # Execute action in environment
+                step_res = requests.post(f"{ENV_URL}/step", json={"action": action_payload})
+                step_data = step_res.json() 
                 
-                # Execute action
-                step_res = requests.post(f"{ENV_URL}/step", json=action_payload)
-                step_data = step_res.json()
-                
-                # Parse response perfectly
+                # Update loop variables
                 observation = step_data.get("observation", {})
                 done = step_data.get("done", False)
                 reward = step_data.get("reward", 0.0)
@@ -112,17 +144,13 @@ def main() -> None:
                 rewards.append(reward)
                 log_step(step=steps_taken, action=action_str, reward=reward, done=done, error=None)
                 
-            # --- THE FIX IS HERE ---
-            # Calculate final score and forcefully clamp it strictly between 0.01 and 0.99
+            # 4. Final Scoring (Single Log)
             raw_score = sum(rewards)
-            score = min(max(raw_score, 0.01), 0.99)
-            success = score > 0.01
-            
-            log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
+            success = raw_score > 0
+            log_end(success=success, steps=steps_taken, score=raw_score, rewards=rewards)
 
         except Exception as e:
             log_step(step=steps_taken+1, action="exception", reward=0.0, done=True, error=str(e).replace("\n", " "))
-            # Forced score to 0.01 instead of 0.0
             log_end(success=False, steps=steps_taken, score=0.01, rewards=rewards)
 
 if __name__ == "__main__":

src/environment.py

@@ -1,90 +1,136 @@
-import uuid
+import requests
 from openenv.core.env_server import Environment
 from src.models import AdAction, AdObservation, AdState
 from src.generator import AdGenerator
 
+REGULATORY_API = "http://localhost:8001"
+CRM_API        = "http://localhost:8002"
+AUDIT_API      = "http://localhost:8003"
+
 class AdPolicyEnvironment(Environment):
     def __init__(self):
         super().__init__()
         self.generator = AdGenerator()
         self.current_ad = None
         self.image_analyzed = False
+        self.regulations_queried = False
+        self.audit_submitted = False
         self.step_count = 0
         self.total_reward = 0.0
 
-    def _ensure_ad(self):
+    def _ensure_ad(self, task_id=None):
         if self.current_ad is None:
-            self.current_ad = self.generator.generate_random_ad()
+            self.current_ad = self.generator.generate_random_ad(task_id)
+            self.current_ad["task_id"] = task_id or "task_1_healthcare"
 
     def state(self) -> AdState:
         self._ensure_ad()
         return AdState(
             step_count=self.step_count,
             total_reward=self.total_reward,
-            current_ad_id=self.current_ad.get("ad_id")
+            current_ad_id=self.current_ad.get("ad_id", "N/A")
         )
 
-    # Add task_id as an optional parameter
     def reset(self, task_id: str = None) -> AdObservation:
-        # Pass the task_id down to the generator
         self.current_ad = self.generator.generate_random_ad(task_id)
+        self.current_ad["task_id"] = task_id or "task_1_healthcare" 
         self.image_analyzed = False
+        self.regulations_queried = False
+        self.audit_submitted = False
         self.step_count = 0
         self.total_reward = 0.0
-        
-        # Add the task_id to the welcome message so the bot knows it worked
-        msg = f"Ad loaded for {task_id}. Awaiting review." if task_id else "Random ad loaded. Awaiting review."
-        return self._get_obs(msg)
+        return self._get_obs(f"Ad loaded for {self.current_ad['task_id']}. Begin with query_regulations.")
 
-    def step(self, action: AdAction) -> AdObservation: 
+    def step(self, action: AdAction) -> AdObservation:
         self._ensure_ad()
         self.step_count += 1
-        
         reward = 0.0
         done = False
-        message = "Action processed."
 
         if not action or not hasattr(action, 'action_type'):
-            # Heavy penalty for invalid formatting to train the agent faster
-            reward = -0.1 
-            self.total_reward += reward
-            return self._get_obs("Invalid action.", reward, False)
+            return self._get_obs("Invalid action format.", -0.1, False)
 
         act_type = str(action.action_type).lower()
+        task_id  = self.current_ad.get("task_id", "")
+
+        # ── TOOL ACTIONS ──────────────────────────────────────────────────────
+        if act_type == "query_regulations":
+            self.regulations_queried = True
+            reward = -0.05
+            category = self.current_ad.get("category", "general")
+            try:
+                resp = requests.get(f"{REGULATORY_API}/regulations/{category}", timeout=2)
+                message = resp.json().get("policy_summary", "Standard policy applies.")
+            except Exception:
+                message = "API Error: Default standard policy applies."
+
+        elif act_type == "analyze_image":
+            self.image_analyzed = True
+            reward = -0.05
+            message = self.current_ad.get("vlm_desc", "No visual anomalies detected.")
+
+        elif act_type == "check_advertiser_history":
+            reward = -0.05
+            advertiser_id = self.current_ad.get("advertiser_id", "adv_003")
+            try:
+                resp = requests.get(f"{CRM_API}/advertiser/{advertiser_id}", timeout=2)
+                message = f"CRM Summary: {resp.json().get('summary', 'No data')}"
+            except Exception:
+                message = "CRM offline. Cannot verify history."
+
+        elif act_type == "request_landing_page":
+            reward = -0.05
+            domain_age = self.current_ad.get("domain_age_days", 365)
+            risk_keywords = self.current_ad.get("landing_risk_keywords", [])
+            message = f"Domain age: {domain_age} days. Flagged terms: {risk_keywords or 'none'}."
 
-        # 🧠 REWARD SHAPING: Tool Usage Penalties
-        # We charge the agent -0.05 for using tools to force efficiency
-        if act_type in ["analyze_image", "request_landing_page", "request_id_verification"]:
+        elif act_type == "request_id_verification":
             reward = -0.05
+            age_min = self.current_ad.get("targeting_data", {}).get("age_min", 18)
+            message = f"Target age {age_min}+." if age_min >= 18 else f"ALERT: Minor targeting detected (Age {age_min}+)."
+
+        elif act_type == "submit_audit":
+            self.audit_submitted = True
+            reward = 0.0 
+            try:
+                payload = {
+                    "ad_id": self.current_ad.get("ad_id", "test"),
+                    "action_taken": "pending",
+                    "reasoning": "audit requested"
+                }
+                resp = requests.post(f"{AUDIT_API}/log", json=payload, timeout=2)
+                message = f"Audit logged: {resp.json().get('audit_id', 'Local-1')}"
+            except Exception:
+                message = "Audit recorded locally."
+
+        # ── TERMINAL ACTIONS (Phase Gates) ────────────────────────────────────
+        elif act_type in ["approve", "reject"]:
+            # Gate 1: Must query rules
+            if not self.regulations_queried:
+                return self._get_obs("Policy Gate: Run query_regulations first.", -0.2, False)
             
-            if act_type == "analyze_image":
-                self.image_analyzed = True
-                vlm_text = self.current_ad.get('vlm_desc', 'No visual description.')
-                message = vlm_text  # Cleaned up the double "VLM Output:" prefix here!
+            # Gate 2: Multimodal tasks require image analysis
+            if "multimodal" in task_id and not self.image_analyzed:
+                return self._get_obs("Visual Gate: Image analysis required.", -0.3, False)
             
-            elif act_type == "request_landing_page":
-                is_bad = self.current_ad.get("ground_truth", False)
-                message = "Landing page is high-risk." if is_bad else "Landing page is clean."
-                
-            elif act_type == "request_id_verification":
-                message = "ID matches advertiser profile."
-
-        # 🎯 FINAL DECISION: Big Rewards / Big Penalties
-        elif act_type in ["approve", "reject"]:
+            # Gate 3: Must audit
+            if not self.audit_submitted:
+                return self._get_obs("Compliance Gate: Run submit_audit before decision.", -0.2, False)
+
             done = True
             is_violation = self.current_ad.get("ground_truth", False)
-            is_correct = (act_type == "reject" and is_violation) or \
-                         (act_type == "approve" and not is_violation)
-            
+            is_correct = (act_type == "reject" and is_violation) or (act_type == "approve" and not is_violation)
             reward = 1.0 if is_correct else -1.0
-            message = f"Decision: {act_type.upper()}. " + ("Correct!" if is_correct else "Incorrect.")
+            message = f"Decision: {act_type.upper()}. {'Correct!' if is_correct else 'Incorrect.'}"
 
-        # Accumulate the total reward for the episode state
-        self.total_reward += reward
+        else:
+            reward = -0.05
+            message = f"Unknown action: {act_type}."
 
+        self.total_reward += reward
         return self._get_obs(message, reward, done)
 
-    def _get_obs(self, message: str, reward: float = 0.0, done: bool = False) -> AdObservation:
+    def _get_obs(self, message, reward=0.0, done=False) -> AdObservation:
         self._ensure_ad()
         return AdObservation(
             ad_id=str(self.current_ad.get("ad_id", "N/A")),
@@ -94,6 +140,6 @@ class AdPolicyEnvironment(Environment):
             targeting_data=dict(self.current_ad.get("targeting_data", {})),
             image_url=str(self.current_ad.get("image_url", "N/A")),
             status_message=str(message),
-            reward=reward, 
-            done=done       
+            reward=reward,
+            done=done
         )

src/meta_ad_policy_sandbox.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: meta-ad-policy-sandbox
+Version: 0.2.3
+Summary: Meta Ad-Policy RL Sandbox
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: pydantic
+Requires-Dist: requests
+Requires-Dist: openai
+Requires-Dist: openenv-core>=0.2.0

src/meta_ad_policy_sandbox.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+src/__init__.py
+src/environment.py
+src/generator.py
+src/models.py
+src/meta_ad_policy_sandbox.egg-info/PKG-INFO
+src/meta_ad_policy_sandbox.egg-info/SOURCES.txt
+src/meta_ad_policy_sandbox.egg-info/dependency_links.txt
+src/meta_ad_policy_sandbox.egg-info/entry_points.txt
+src/meta_ad_policy_sandbox.egg-info/requires.txt
+src/meta_ad_policy_sandbox.egg-info/top_level.txt

src/meta_ad_policy_sandbox.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/meta_ad_policy_sandbox.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = server.app:main

src/meta_ad_policy_sandbox.egg-info/requires.txt

@@ -0,0 +1,6 @@
+fastapi
+uvicorn
+pydantic
+requests
+openai
+openenv-core>=0.2.0

src/meta_ad_policy_sandbox.egg-info/top_level.txt

@@ -0,0 +1,4 @@
+__init__
+environment
+generator
+models