""" inference.py - AI Executive Operations Manager Runs all 3 tasks using an LLM agent and prints structured logs. Environment variables: API_BASE_URL - OpenAI-compatible API base URL MODEL_NAME - Model to use (e.g., "gpt-4o-mini", "meta-llama/Llama-3.1-8B-Instruct") HF_TOKEN - API key / HuggingFace token STDOUT format (strict): [START] task= env=exec-ops model= [STEP] step= action=('') reward=<0.00> done= error= [END] success= steps= score=<0.00> rewards= """ import os import json import sys import dotenv # Force UTF-8 output on Windows if sys.stdout.encoding != "utf-8": sys.stdout.reconfigure(encoding="utf-8") from openai import OpenAI from env import ExecOpsEnv, Action from env.grader import grade # ------------------------------------------------------- # Configuration from environment # ------------------------------------------------------- dotenv.load_dotenv() API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1") MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini") HF_TOKEN = os.getenv("HF_TOKEN") LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME") client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN) SYSTEM_PROMPT = """You are an expert AI assistant helping a startup CEO (Alex Rivera at NovaTech AI) manage their inbox efficiently. Your job: Given the current environment state, choose ONE action for ONE email. Triage principles: 1. Handle the highest PRIORITY * URGENCY emails first 2. Reply to items requiring personal CEO attention (investor relations, critical incidents, legal deadlines) 3. Delegate items that can be handled by team members (HR, finance, operations) 4. Schedule items requiring a meeting 5. Ignore truly trivial items only when critical items remain You MUST respond with ONLY valid JSON, no explanation: {"type": "reply|schedule|delegate|ignore", "email_id": ""} IMPORTANT: email_id MUST be one of the ids from the "unhandled_emails" list (e.g. "e1", "e2"). Do NOT use goal_id values (e.g. "g1", "g2") as the email_id. Choose wisely - you have limited steps. Ignoring P4-P5 items is heavily penalized.""" def _log(msg: str) -> None: """Print a structured log line to stdout.""" print(msg, flush=True) def _err(msg: str) -> None: """Print diagnostic info to stderr - never pollutes stdout.""" print(msg, file=sys.stderr, flush=True) def _clean_error(msg: str) -> str: """Collapse error message to a single line safe for the log format.""" return msg.replace("\n", " ").replace("\r", "").strip()[:120] def parse_action_from_response(text: str) -> dict: """Extract JSON action from LLM response, handling markdown code blocks.""" text = text.strip() if "```json" in text: text = text.split("```json")[1].split("```")[0].strip() elif "```" in text: parts = text.split("```") for part in parts[1::2]: part = part.strip() if part.startswith("{"): text = part break start = text.find("{") end = text.rfind("}") + 1 if start >= 0 and end > start: text = text[start:end] return json.loads(text) def get_fallback_action(obs: dict) -> Action: """Fallback: pick highest priority * urgency unhandled email and reply or delegate.""" inbox = obs.get("inbox", []) unhandled = [e for e in inbox if not e.get("handled", False)] if not unhandled: return None best = max(unhandled, key=lambda e: e.get("priority", 1) * e.get("urgency", 0.5)) action_type = "delegate" if best.get("priority", 1) <= 2 else "reply" return Action(type=action_type, email_id=best["id"]) def run_task(task_id: str) -> float: """Run a single task with the LLM agent. Returns final grade in [0, 1].""" _log(f"[START] task={task_id} env=exec-ops model={MODEL_NAME}") step_count = 0 rewards: list[float] = [] pending_step_log = None # deferred so we can force done=true on the last step final_score = 0.0 env = None try: env = ExecOpsEnv(task_id) obs = env.reset() max_steps = obs.get("max_steps", 10) while not obs.get("done", False) and step_count < max_steps: unhandled = [e for e in obs.get("inbox", []) if not e.get("handled", False)] if not unhandled: break # Flush previous buffered step - it is not the last, so done=false if pending_step_log is not None: _log(pending_step_log.replace("_DONE_", "false")) pending_step_log = None prompt_state = { "time": obs.get("time"), "steps_remaining": obs.get("steps_remaining", max_steps - step_count), "unhandled_emails": [ { "id": e["id"], "sender": e["sender"].split("<")[0].strip(), "subject": e["subject"], "priority": e["priority"], "urgency": round(e["urgency"], 2), } for e in sorted(unhandled, key=lambda x: -(x["priority"] * x["urgency"])) ], "pending_goals": [ {"goal_id": g["id"], "description": g["description"], "priority": g["priority"]} for g in obs.get("pending_goals", []) ], } # --- Get action from LLM (with fallback) --- step_error = None action = None try: response = client.chat.completions.create( model=MODEL_NAME, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": f"Current state:\n{json.dumps(prompt_state, indent=2)}\n\nChoose your action (JSON only):"} ], temperature=0.1, max_tokens=80, ) action_text = response.choices[0].message.content action_data = parse_action_from_response(action_text) action = Action(type=action_data["type"], email_id=action_data["email_id"]) valid_ids = {e["id"] for e in unhandled} if action.email_id not in valid_ids: raise ValueError(f"Invalid email_id: {action.email_id}") except Exception as e: step_error = _clean_error(str(e)) _err(f"[fallback] LLM error ({task_id} step {step_count+1}): {type(e).__name__}: {e}") action = get_fallback_action(obs) if action is None: break # --- Execute step; retry with fallback on failure --- candidates = [action] fb = get_fallback_action(obs) if fb is not None and (action is None or fb.email_id != action.email_id): candidates.append(fb) executed = False for attempt in candidates: try: result = env.step(attempt) reward = result.get("reward", 0.0) done = result.get("done", False) obs = result.get("observation", result) if "done" not in obs: obs["done"] = done step_count += 1 rewards.append(reward) action_str = f"{attempt.type}('{attempt.email_id}')" error_field = step_error if step_error else "null" pending_step_log = ( f"[STEP] step={step_count} action={action_str} " f"reward={reward:.2f} done=_DONE_ error={error_field}" ) executed = True break except Exception as e: step_error = _clean_error(str(e)) _err(f"[fallback] Step error ({task_id} step {step_count+1}): {type(e).__name__}: {e}") if not executed: break except Exception as e: _err(f"[error] Unexpected error in task '{task_id}': {e}") finally: # Flush the final buffered step - always done=true if pending_step_log is not None: _log(pending_step_log.replace("_DONE_", "true")) try: final_score = grade(env._state) if env is not None else 0.0 except Exception: final_score = 0.0 success = "true" if final_score >= 0.5 else "false" rewards_str = ",".join(f"{r:.2f}" for r in rewards) if rewards else "0.00" _log(f"[END] success={success} steps={step_count} score={final_score:.3f} rewards={rewards_str}") return final_score def main(): tasks = ["easy", "medium", "hard"] scores = {} for task_id in tasks: try: score = run_task(task_id) scores[task_id] = score except Exception as e: _err(f"[error] Task '{task_id}' failed: {e}") scores[task_id] = 0.0 _log(f"[END] success=false steps=0 score=0.000 rewards=0.00") return sum(scores.values()) / len(scores) if scores else 0.0 if __name__ == "__main__": avg = main() sys.exit(0)