""" inference.py — Meta OpenEnv Hackathon Submission v3.2 24 tasks: 5 domains × 4 (easy/medium/hard/expert) + 4 cross-agent chained tasks. Provider priority (auto-detected from env vars): 1. GROQ_API_KEY → https://api.groq.com/openai/v1 (llama-3.3-70b-versatile) [FREE] 2. HF_TOKEN → API_BASE_URL (Qwen/Qwen2.5-72B-Instruct) 3. OPENAI_API_KEY → https://api.openai.com/v1 (gpt-4o-mini) Logging format: [START] task= env=meta model= [STEP] step= action= reward=<0.00> done= error= [END] success= steps= score=<0.00> rewards= v3.2: handles both TPM (per-minute) and TPD (per-day) Groq rate limits correctly. TPM → short backoff and retry. TPD → wait for the exact reset time reported in the error, then retry. """ from __future__ import annotations import json import os import re import sys import time from typing import Optional # ── Load .env ───────────────────────────────────────────────────────────────── _env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env") if os.path.exists(_env_file): with open(_env_file, encoding="utf-8", errors="ignore") as _f: for _line in _f: _line = _line.strip() if _line and not _line.startswith("#") and "=" in _line: _k, _, _v = _line.partition("=") os.environ.setdefault(_k.strip(), _v.strip()) # ── Provider resolution ─────────────────────────────────────────────────────── _groq_key = os.environ.get("GROQ_API_KEY", "") _hf_token = os.environ.get("HF_TOKEN", "") _openai_key = os.environ.get("OPENAI_API_KEY", "") _api_base = os.environ.get("API_BASE_URL", "") _model_name = os.environ.get("MODEL_NAME", "") if _groq_key: API_BASE_URL = "https://api.groq.com/openai/v1" MODEL_NAME = _model_name or "llama-3.3-70b-versatile" API_KEY = _groq_key PROVIDER = "groq" elif _hf_token: API_BASE_URL = _api_base or "https://router.huggingface.co/v1" MODEL_NAME = _model_name or "Qwen/Qwen2.5-72B-Instruct" API_KEY = _hf_token PROVIDER = "huggingface" elif _openai_key: API_BASE_URL = "https://api.openai.com/v1" MODEL_NAME = _model_name or "gpt-4o-mini" API_KEY = _openai_key PROVIDER = "openai" else: API_BASE_URL = _api_base or "https://router.huggingface.co/v1" MODEL_NAME = _model_name or "Qwen/Qwen2.5-72B-Instruct" API_KEY = "" PROVIDER = "none" HF_TOKEN = API_KEY TASK_IDS = [ "email_triage_easy", "email_triage_medium", "email_triage_hard", "email_triage_expert", "code_review_easy", "code_review_medium", "code_review_hard", "code_review_expert", "data_cleaning_easy", "data_cleaning_medium", "data_cleaning_hard", "data_cleaning_expert", "content_moderation_easy", "content_moderation_medium", "content_moderation_hard", "content_moderation_expert", "ticket_triage_easy", "ticket_triage_medium", "ticket_triage_hard", "ticket_triage_expert", "cross_agent_chain", "cross_agent_email_data", "cross_agent_code_email", "cross_agent_mod_escalation", ] SYSTEM_PROMPT = ( "You are an expert AI agent completing structured real-world tasks.\n" "You MUST respond with ONLY a valid JSON object as the payload.\n" "Rules:\n" "1. Start your response with { and end with }\n" "2. No markdown, no code fences (no ```), no explanation text\n" "3. No preamble — just the JSON object\n" "4. The JSON must be valid and parseable\n" "5. Follow the payload format exactly as described in the instructions" ) SUCCESS_SCORE_THRESHOLD = 0.5 # Groq free tier: 30 req/min TPM limit → 2s is safe per request # TPD limit is 100k tokens/day — handled separately via TPD detection GROQ_SLEEP = 2.0 DEFAULT_SLEEP = 1.0 # Max seconds to wait for TPD reset before giving up (20 min) MAX_TPD_WAIT = 1200 # ── Rate limit parsing ──────────────────────────────────────────────────────── def _parse_wait_seconds(err_str: str) -> float: """Extract the suggested wait time in seconds from a Groq 429 error message.""" # Format: "Please try again in 12m50.688s" or "56.16s" or "1m5.664s" match = re.search(r"try again in ([\d.]+)m([\d.]+)s", err_str) if match: return float(match.group(1)) * 60 + float(match.group(2)) match = re.search(r"try again in ([\d.]+)s", err_str) if match: return float(match.group(1)) return 0.0 def _is_tpd_limit(err_str: str) -> bool: """Returns True if the error is a tokens-per-day limit (not per-minute).""" return "tokens per day" in err_str.lower() or "tpd" in err_str.lower() # ── Logging ─────────────────────────────────────────────────────────────────── def log_start(task: str, env: str, model: str) -> None: print(f"[START] task={task} env={env} model={model}", flush=True) def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None: error_val = error if error else "null" done_val = str(done).lower() action_oneline = action.replace("\n", " ")[:120] reward_clamped = round(max(1e-4, min(1.0 - 1e-4, float(reward))), 4) print(f"[STEP] step={step} action={action_oneline} reward={reward_clamped:.4f} done={done_val} error={error_val}", flush=True) def _clamp_score(score: float) -> float: """Clamp score to open interval (0, 1) — evaluator rejects 0.0 and 1.0 exactly.""" _EPS = 1e-4 return round(max(_EPS, min(1.0 - _EPS, float(score))), 4) def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None: score = _clamp_score(score) rewards_clamped = [_clamp_score(r) for r in rewards] rewards_str = ",".join(f"{r:.4f}" for r in rewards_clamped) print(f"[END] success={str(success).lower()} steps={steps} score={score:.4f} rewards={rewards_str}", flush=True) # ── Helpers ─────────────────────────────────────────────────────────────────── def progress_bar(current: int, total: int, width: int = 40) -> str: filled = int(width * current / total) return f"[{'█'*filled}{'░'*(width-filled)}] {int(100*current/total):3d}% ({current}/{total})" def score_bar(score: float, width: int = 10) -> str: filled = int(width * score) return "█" * filled + "░" * (width - filled) def extract_json(text: str) -> dict: if not text: return {} text = text.strip() try: r = json.loads(text) if isinstance(r, dict): return r except Exception: pass cleaned = re.sub(r"^```(?:json)?\s*\n?", "", text, flags=re.IGNORECASE) cleaned = re.sub(r"\n?```\s*$", "", cleaned, flags=re.MULTILINE).strip() try: r = json.loads(cleaned) if isinstance(r, dict): return r except Exception: pass start = text.find("{") if start != -1: depth = 0 for i, ch in enumerate(text[start:], start): if ch == "{": depth += 1 elif ch == "}": depth -= 1 if depth == 0: try: r = json.loads(text[start:i+1]) if isinstance(r, dict): return r except Exception: pass break return {} def unwrap_payload(payload: dict) -> dict: if payload and "payload" in payload and isinstance(payload["payload"], dict): return payload["payload"] return payload def call_model(client, task_id: str, difficulty: str, instructions: str, agent_ctx: dict, feedback: Optional[str] = None, is_groq: bool = False) -> dict: """ Call the model with adaptive retry on rate limit errors. Distinguishes between TPM (per-minute, short wait) and TPD (per-day, long wait). """ retry_note = "" if feedback: retry_note = ( f"\nPREVIOUS ATTEMPT FEEDBACK: {feedback}\n" "Fix the issues described above in your new response.\n" ) user_prompt = ( f"Task: {task_id}\n" f"Difficulty: {difficulty}\n\n" f"Instructions:\n{instructions}\n" f"{retry_note}\n" f"Data to work with:\n{json.dumps(agent_ctx, indent=2)}\n\n" "Respond with ONLY the payload JSON object. Start with { and end with }." ) max_api_retries = 6 tpm_backoff = 4.0 for attempt in range(max_api_retries): try: response = client.chat.completions.create( model=MODEL_NAME, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}, ], temperature=0.0, max_tokens=2000, ) raw = response.choices[0].message.content or "" payload = unwrap_payload(extract_json(raw)) return payload except Exception as e: err_str = str(e) if "429" in err_str or "rate_limit" in err_str.lower(): wait_secs = _parse_wait_seconds(err_str) if _is_tpd_limit(err_str): # Tokens-per-day exhausted — must wait for daily reset if wait_secs > MAX_TPD_WAIT: print(f"\n [TPD LIMIT] Daily token limit exhausted. " f"Reset in {wait_secs/60:.1f}min — exceeds max wait. Skipping task.") return {} wait_secs = max(wait_secs + 5, 60) # add 5s buffer print(f"\n [TPD LIMIT] Daily token limit exhausted. " f"Waiting {wait_secs/60:.1f}min for reset (attempt {attempt+1}/{max_api_retries})...") time.sleep(wait_secs) else: # TPM limit — short wait wait_time = max(wait_secs + 0.5, tpm_backoff) if wait_secs > 0 else tpm_backoff wait_time = min(wait_time, 60.0) print(f"\n [RATE LIMIT] Waiting {wait_time:.1f}s before retry " f"{attempt+1}/{max_api_retries}...") time.sleep(wait_time) tpm_backoff = min(tpm_backoff * 1.5, 60.0) else: print(f"\n [ERROR] API call failed: {e}") return {} print(f"\n [ERROR] All {max_api_retries} retries exhausted.") return {} # ── Single task episode ─────────────────────────────────────────────────────── def run_task(client, task_id: str, INSTRUCTIONS: dict, is_groq: bool = False) -> dict: from server.Meta_environment import MetaEnvironment from models import MetaAction agent = ( "_".join(task_id.split("_")[:-1]) if not task_id.startswith("cross_agent") else "cross_agent" ) env = MetaEnvironment() env.reset() try: agent_ctx, grader_ctx, difficulty = env._load_context(task_id) except Exception as e: return { "task_id": task_id, "agent": agent, "difficulty": "unknown", "score": 1e-4, "feedback": f"Context load failed: {e}", "partial_credits": {}, "reward": 1e-4, } instructions = INSTRUCTIONS.get(task_id, "") env2 = MetaEnvironment() env2.reset() env2._current_agent_context = agent_ctx env2._current_grader_context = grader_ctx env2._current_task_id = task_id rewards: list[float] = [] log_start(task_id, "meta", MODEL_NAME) # Attempt 1 payload = call_model(client, task_id, difficulty, instructions, agent_ctx, is_groq=is_groq) action = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": payload})) obs = env2.step(action) rewards.append(obs.reward) log_step(1, json.dumps(payload)[:80], obs.reward, obs.done, None) # Attempt 2 if retry available if not obs.done and obs.score < 1.0: print(f"\n [RETRY] Score {obs.score:.2f} — retrying with feedback...") time.sleep(GROQ_SLEEP if is_groq else 1.0) payload2 = call_model(client, task_id, difficulty, instructions, agent_ctx, feedback=obs.feedback, is_groq=is_groq) action2 = MetaAction(message=json.dumps({"agent": agent, "task_id": task_id, "payload": payload2})) obs = env2.step(action2) rewards.append(obs.reward) log_step(2, json.dumps(payload2)[:80], obs.reward, obs.done, None) final_score = _clamp_score(obs.score) success = final_score >= SUCCESS_SCORE_THRESHOLD log_end(success, len(rewards), final_score, rewards) return { "task_id": task_id, "agent": agent, "difficulty": obs.difficulty, "score": final_score, "feedback": obs.feedback, "partial_credits": obs.partial_credits, "reward": obs.reward, } # ── Main ────────────────────────────────────────────────────────────────────── def run_all(api_base_url: str, model_name: str, hf_token: str) -> list[dict]: api_key = hf_token or API_KEY if not api_key: print("ERROR: No API key found.") print(" Set one of:") print(" GROQ_API_KEY=gsk_... (free at https://console.groq.com)") print(" HF_TOKEN=hf_... (HuggingFace token)") print(" OPENAI_API_KEY=sk-... (OpenAI)") sys.exit(1) try: from openai import OpenAI except ImportError: print("ERROR: pip install openai") sys.exit(1) try: sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from server.Meta_environment import INSTRUCTIONS except ImportError as e: print(f"ERROR: {e}") sys.exit(1) is_groq = "groq.com" in api_base_url task_sleep = GROQ_SLEEP if is_groq else DEFAULT_SLEEP client = OpenAI(api_key=api_key, base_url=api_base_url) results: list[dict] = [] print() print("╔══════════════════════════════════════════════════════════════════╗") print("║ Meta OpenEnv — Inference Runner (OpenEnv Hackathon 2026) ║") print("╠══════════════════════════════════════════════════════════════════╣") print(f"║ Provider : {PROVIDER:<53}║") print(f"║ API Base : {api_base_url:<53}║") print(f"║ Model : {model_name:<53}║") print(f"║ Tasks : {len(TASK_IDS):<53}║") print(f"║ Sleep : {task_sleep}s between tasks (TPM + TPD limit handling) ║") print("╚══════════════════════════════════════════════════════════════════╝") print() start_all = time.time() for i, task_id in enumerate(TASK_IDS): print(f" {progress_bar(i, len(TASK_IDS))} {task_id}") sys.stdout.flush() t0 = time.time() result = run_task(client, task_id, INSTRUCTIONS, is_groq=is_groq) elapsed = time.time() - t0 results.append(result) diff = result["difficulty"].upper()[:4] bar = score_bar(result["score"]) print(f" ✓ [{diff:4s}] {task_id:<40} [{bar}] {result['score']:.2f} ({elapsed:.1f}s) {result['feedback'][:50]}") sys.stdout.flush() time.sleep(task_sleep) total_time = time.time() - start_all avg_score = sum(r["score"] for r in results) / len(results) passed = sum(1 for r in results if r["score"] >= SUCCESS_SCORE_THRESHOLD) print() print(f" {progress_bar(len(TASK_IDS), len(TASK_IDS))}") print() print("╔══════════════════════════════════════════════════════════════════╗") print(f"║ 📊 Average Score : {avg_score:.4f}{'':37}║") print(f"║ ⏱ Total Time : {total_time:.1f}s{'':39}║") print(f"║ ✅ Tasks Passed : {passed}/{len(results)}{'':42}║") print("╚══════════════════════════════════════════════════════════════════╝") print() agents = ["email_triage","code_review","data_cleaning","content_moderation","ticket_triage","cross_agent"] print(" Per-Agent Summary:") for ag in agents: ag_r = [r for r in results if r["agent"] == ag] ag_avg = sum(r["score"] for r in ag_r) / len(ag_r) if ag_r else 0.0 print(f" {ag:<35} [{score_bar(ag_avg)}] {ag_avg:.2f}") print() return results if __name__ == "__main__": results = run_all(API_BASE_URL, MODEL_NAME, HF_TOKEN) print("Full Results JSON:") print(json.dumps(results, indent=2))