commit b73fc11a8c99354df8c68bac26d7338aa0f40313 Author: VISHAL_V Date: Tue Apr 7 14:33:13 2026 +0530 fix: enforce strict (0,1) score range with centralized clamp_score validator - add Pydantic field_validator on Reward.score to auto-clamp to (0.01, 0.99) - use centralized clamp_score in graders.py and environment.py - clamp scores in inference.py before printing diff --git a/inference.py b/inference.py index 2bdb4d1..67aaf17 100644 --- a/inference.py +++ b/inference.py @@ -131,8 +131,9 @@ def run_task(task_id: str) -> float: done = response.get("done", True) score_now = reward_dict.get("score", 0.01) if isinstance(reward_dict, dict) else 0.01 + score_now = max(0.01, min(0.99, float(score_now))) success = done and score_now > 0.5 - print(f"[STEP] reward={score_now:.2f} done={str(done).lower()} success={str(success).lower()}") + print(f"[STEP] reward={score_now:.4f} done={str(done).lower()} success={str(success).lower()}") print("[END]") final_score = reward_dict.get("score", 0.01) if isinstance(reward_dict, dict) else 0.01 commit b5e447a011fb71978124db2d94ad0aeaa74c8285 Author: VISHAL_V Date: Tue Apr 7 12:51:29 2026 +0530 Fix score bounds: clamp to (0.01, 0.99) instead of [0.0, 1.0] diff --git a/inference.py b/inference.py index d9fbbc6..2bdb4d1 100644 --- a/inference.py +++ b/inference.py @@ -62,7 +62,7 @@ def step_env(action: dict) -> dict: print(f"[ERROR] Step failed: {e}", file=sys.stderr) return { "observation": {}, - "reward": {"score": 0.0, "breakdown": {}}, + "reward": {"score": 0.01, "breakdown": {}}, "done": True, "info": {"error": str(e)} } @@ -113,13 +113,13 @@ def run_task(task_id: str) -> float: obs = reset_env(task_id) if not obs: - print("[STEP] reward=0.00 done=true success=false") + print("[STEP] reward=0.01 done=true success=false") print("[END]") - return 0.0 + return 0.01 done = False steps = 0 - reward_dict: dict = {"score": 0.0} + reward_dict: dict = {"score": 0.01} while not done and steps < MAX_STEPS: steps += 1 @@ -127,15 +127,16 @@ def run_task(task_id: str) -> float: response = step_env(action) obs = response.get("observation", {}) - reward_dict = response.get("reward", {"score": 0.0}) + reward_dict = response.get("reward", {"score": 0.01}) done = response.get("done", True) - score_now = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 + score_now = reward_dict.get("score", 0.01) if isinstance(reward_dict, dict) else 0.01 success = done and score_now > 0.5 print(f"[STEP] reward={score_now:.2f} done={str(done).lower()} success={str(success).lower()}") print("[END]") - final_score = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 + final_score = reward_dict.get("score", 0.01) if isinstance(reward_dict, dict) else 0.01 + final_score = max(0.01, min(0.99, final_score)) return final_score commit 4d30c103b600a290bbdd3f9d8fe494fbb358f87e Author: VISHAL_V Date: Tue Apr 7 10:55:44 2026 +0530 fix: inference.py stdout format compliance for [START]/[STEP]/[END] spec diff --git a/inference.py b/inference.py index da78eb6..d9fbbc6 100644 --- a/inference.py +++ b/inference.py @@ -1,4 +1,5 @@ import os +import sys import json import requests from typing import Dict, Any @@ -47,7 +48,7 @@ def reset_env(task_id: str) -> dict: response.raise_for_status() return response.json() except Exception as e: - print(f"[ERROR] Reset failed: {e}") + print(f"[ERROR] Reset failed: {e}", file=sys.stderr) return {} @@ -58,7 +59,7 @@ def step_env(action: dict) -> dict: response.raise_for_status() return response.json() except Exception as e: - print(f"[ERROR] Step failed: {e}") + print(f"[ERROR] Step failed: {e}", file=sys.stderr) return { "observation": {}, "reward": {"score": 0.0, "breakdown": {}}, @@ -103,18 +104,17 @@ def get_action_from_llm(obs: dict) -> dict: return action_obj except Exception as e: - print(f"[ERROR] LLM call failed: {e}") + print(f"[ERROR] LLM call failed: {e}", file=sys.stderr) return dict(NOOP_ACTION) def run_task(task_id: str) -> float: - print(f"\n{'='*40}") - print(f"Starting {task_id}") - print(f"{'='*40}") + print("[START]") obs = reset_env(task_id) if not obs: - print(f"[WARN] Failed to initialize environment for {task_id}.") + print("[STEP] reward=0.00 done=true success=false") + print("[END]") return 0.0 done = False @@ -124,20 +124,18 @@ def run_task(task_id: str) -> float: while not done and steps < MAX_STEPS: steps += 1 action = get_action_from_llm(obs) - print(f" Step {steps} | Action: {action}") response = step_env(action) obs = response.get("observation", {}) reward_dict = response.get("reward", {"score": 0.0}) done = response.get("done", True) - info = response.get("info", {}) score_now = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 - print(f" Score: {score_now} | Info: {info}") - print(f" {'-'*36}") + success = done and score_now > 0.5 + print(f"[STEP] reward={score_now:.2f} done={str(done).lower()} success={str(success).lower()}") + print("[END]") final_score = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 - print(f"Task {task_id} complete. Final score: {final_score}") return final_score @@ -149,16 +147,3 @@ if __name__ == "__main__": score = run_task(task) scores[task] = score total_score += score - - print(f"\n\n{'='*40}") - print("FINAL RESULTS") - print(f"{'='*40}") - for t, s in scores.items(): - print(f" {t}: {s}") - - avg_score = total_score / len(TASKS_TO_RUN) if TASKS_TO_RUN else 0.0 - print(f"\nAverage Score: {avg_score:.4f}") - if avg_score >= 0.8: - print("Result: PASS ΓÇö Agent performed well.") - else: - print("Result: NEEDS IMPROVEMENT ΓÇö Agent performance is below threshold.") commit 82b9c7a8a7b18863cb7a66fe0621d20a5f321d85 Author: VISHAL_V Date: Fri Apr 3 12:31:09 2026 +0530 Final Hackathon Submission diff --git a/inference.py b/inference.py new file mode 100644 index 0000000..da78eb6 --- /dev/null +++ b/inference.py @@ -0,0 +1,164 @@ +import os +import json +import requests +from typing import Dict, Any +from openai import OpenAI + +API_BASE_URL = os.getenv("API_BASE_URL") +HF_TOKEN = os.getenv("HF_TOKEN") +MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini") +ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860") + +client_kwargs = {"api_key": HF_TOKEN} +if API_BASE_URL: + client_kwargs["base_url"] = API_BASE_URL + +client = OpenAI(**client_kwargs) + +TASKS_TO_RUN = ["task_1", "task_2", "task_3"] +MAX_STEPS = 8 +REQUEST_TIMEOUT = 30 + +SYSTEM_PROMPT = """You are an Enterprise Email Agent. +Your goal is to handle incoming emails appropriately. +Allowed Actions: classify_email, route_to, draft_reply, escalate, mark_spam, request_more_info, noop. + +You MUST respond strictly in valid JSON format containing two keys: "action_type" and "arguments". + +Example 1: +{"action_type": "classify_email", "arguments": {"category": "support"}} + +Example 2: +{"action_type": "route_to", "arguments": {"department": "sales_department"}} + +Example 3: +{"action_type": "draft_reply", "arguments": {"text": "We are looking into this immediate issue."}} + +Always choose the most relevant action. Avoid unnecessary actions. +""" + +NOOP_ACTION = {"action_type": "noop", "arguments": {}} + + +def reset_env(task_id: str) -> dict: + url = f"{ENV_BASE_URL}/reset" + try: + response = requests.post(url, json={"task_id": task_id}, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + return response.json() + except Exception as e: + print(f"[ERROR] Reset failed: {e}") + return {} + + +def step_env(action: dict) -> dict: + url = f"{ENV_BASE_URL}/step" + try: + response = requests.post(url, json=action, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + return response.json() + except Exception as e: + print(f"[ERROR] Step failed: {e}") + return { + "observation": {}, + "reward": {"score": 0.0, "breakdown": {}}, + "done": True, + "info": {"error": str(e)} + } + + +def get_action_from_llm(obs: dict) -> dict: + prompt = f"Current Observation: {json.dumps(obs, indent=2)}\n\nWhat action will you take next? Reply ONLY with JSON." + + try: + completion = client.chat.completions.create( + model=MODEL_NAME, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt} + ], + temperature=0.0, + max_tokens=150, + response_format={"type": "json_object"} + ) + + reply = completion.choices[0].message.content + if not reply: + return dict(NOOP_ACTION) + + try: + action_obj = json.loads(reply) + except (json.JSONDecodeError, TypeError): + return dict(NOOP_ACTION) + + if not isinstance(action_obj, dict): + return dict(NOOP_ACTION) + + if "action_type" not in action_obj or not isinstance(action_obj["action_type"], str): + return dict(NOOP_ACTION) + + if "arguments" not in action_obj or not isinstance(action_obj["arguments"], dict): + action_obj["arguments"] = {} + + return action_obj + + except Exception as e: + print(f"[ERROR] LLM call failed: {e}") + return dict(NOOP_ACTION) + + +def run_task(task_id: str) -> float: + print(f"\n{'='*40}") + print(f"Starting {task_id}") + print(f"{'='*40}") + + obs = reset_env(task_id) + if not obs: + print(f"[WARN] Failed to initialize environment for {task_id}.") + return 0.0 + + done = False + steps = 0 + reward_dict: dict = {"score": 0.0} + + while not done and steps < MAX_STEPS: + steps += 1 + action = get_action_from_llm(obs) + print(f" Step {steps} | Action: {action}") + + response = step_env(action) + obs = response.get("observation", {}) + reward_dict = response.get("reward", {"score": 0.0}) + done = response.get("done", True) + info = response.get("info", {}) + + score_now = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 + print(f" Score: {score_now} | Info: {info}") + print(f" {'-'*36}") + + final_score = reward_dict.get("score", 0.0) if isinstance(reward_dict, dict) else 0.0 + print(f"Task {task_id} complete. Final score: {final_score}") + return final_score + + +if __name__ == "__main__": + total_score = 0.0 + scores: Dict[str, float] = {} + + for task in TASKS_TO_RUN: + score = run_task(task) + scores[task] = score + total_score += score + + print(f"\n\n{'='*40}") + print("FINAL RESULTS") + print(f"{'='*40}") + for t, s in scores.items(): + print(f" {t}: {s}") + + avg_score = total_score / len(TASKS_TO_RUN) if TASKS_TO_RUN else 0.0 + print(f"\nAverage Score: {avg_score:.4f}") + if avg_score >= 0.8: + print("Result: PASS ΓÇö Agent performed well.") + else: + print("Result: NEEDS IMPROVEMENT ΓÇö Agent performance is below threshold.")