fix: pinned nanobind, disabled benchmark for docker, and isolated state endpoint
Browse files- inference.py +48 -46
- server/app.py +2 -4
- server/fin_auditor_environment.py +13 -33
inference.py
CHANGED
|
@@ -1,19 +1,10 @@
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
-
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
| 3 |
-
# inference.py — OpenEnv Evaluation Script (Hackathon Submission)
|
| 4 |
-
#
|
| 5 |
-
# STDOUT FORMAT (strict regex compliance):
|
| 6 |
-
# [START] task=<task_name> env=<benchmark> model=<model_name>
|
| 7 |
-
# [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
|
| 8 |
-
# [END] success=<true|false> steps=<n> rewards=<r1,r2,...,rn>
|
| 9 |
-
#
|
| 10 |
-
# ALL debug output goes to stderr. NO JSON on stdout. NO extra whitespace.
|
| 11 |
-
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
| 12 |
|
| 13 |
import os
|
| 14 |
import sys
|
| 15 |
import json
|
| 16 |
import re
|
|
|
|
| 17 |
import traceback
|
| 18 |
import time
|
| 19 |
from typing import List
|
|
@@ -36,10 +27,6 @@ except ImportError:
|
|
| 36 |
|
| 37 |
from models import AuditorAction
|
| 38 |
|
| 39 |
-
# ── Debug logger: ONLY to stderr ─────────────────────────────────────────────
|
| 40 |
-
def _dbg(msg: str) -> None:
|
| 41 |
-
print(msg, file=sys.stderr, flush=True)
|
| 42 |
-
|
| 43 |
class LLMResponse(BaseModel):
|
| 44 |
reasoning: str
|
| 45 |
decisions: List[int]
|
|
@@ -52,9 +39,7 @@ if not HF_TOKEN:
|
|
| 52 |
raise ValueError("CRITICAL: HF_TOKEN environment variable is missing.")
|
| 53 |
|
| 54 |
TASK_ID: str = os.getenv("TASK_ID", "anomaly_detection_hard")
|
| 55 |
-
ENV_NAME: str = "fin_auditor"
|
| 56 |
|
| 57 |
-
# FIX: Sync the inference max_steps default with the active task
|
| 58 |
if "easy" in TASK_ID.lower():
|
| 59 |
_DEFAULT_MAX = 5
|
| 60 |
elif "medium" in TASK_ID.lower():
|
|
@@ -86,6 +71,9 @@ Example:
|
|
| 86 |
{"reasoning": "Trade 1 has high risk. Trade 2 is safe.", "decisions": [1, 0, 1]}
|
| 87 |
"""
|
| 88 |
|
|
|
|
|
|
|
|
|
|
| 89 |
def _build_user_prompt(step: int, features: list[list[float]]) -> str:
|
| 90 |
lines = [
|
| 91 |
f"Step {step}: You have {len(features)} flagged trades to audit.",
|
|
@@ -165,10 +153,8 @@ def _call_llm(step: int, features: list[list[float]]) -> list[int]:
|
|
| 165 |
content = response.choices[0].message.content or ""
|
| 166 |
return _parse_llm_decisions(content, len(features))
|
| 167 |
except Exception as e:
|
| 168 |
-
_dbg(f"[LLM RETRY {attempt+1}/{max_retries}] {e}")
|
| 169 |
time.sleep(1)
|
| 170 |
|
| 171 |
-
_dbg("[LLM] All retries exhausted, using risk_score fallback")
|
| 172 |
fallback_decisions = []
|
| 173 |
for row in features:
|
| 174 |
if len(row) >= 4:
|
|
@@ -179,59 +165,75 @@ def _call_llm(step: int, features: list[list[float]]) -> list[int]:
|
|
| 179 |
return fallback_decisions
|
| 180 |
|
| 181 |
def run_inference() -> None:
|
|
|
|
|
|
|
| 182 |
steps_completed: int = 0
|
| 183 |
-
|
| 184 |
-
success: bool = False
|
| 185 |
-
error_msg: str | None = None
|
| 186 |
-
|
| 187 |
-
# ── [START] — always emitted ──────────────────────────────────────────
|
| 188 |
-
print(f"[START] task={TASK_ID} env={ENV_NAME} model={MODEL_NAME}", flush=True)
|
| 189 |
|
| 190 |
try:
|
| 191 |
env = FinAuditorEnvironment()
|
| 192 |
obs = env.reset()
|
|
|
|
| 193 |
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
for step_num in range(1, MAX_STEPS + 1):
|
| 197 |
-
step_reward = 0.0
|
| 198 |
features = obs.features
|
| 199 |
|
| 200 |
if not features:
|
| 201 |
action = AuditorAction(decisions=[])
|
| 202 |
-
|
| 203 |
else:
|
| 204 |
decisions = _call_llm(step_num, features)
|
| 205 |
action = AuditorAction(decisions=decisions)
|
| 206 |
|
| 207 |
obs = env.step(action)
|
| 208 |
-
|
| 209 |
-
|
|
|
|
| 210 |
steps_completed = step_num
|
| 211 |
|
| 212 |
-
#
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
|
| 219 |
if obs.done:
|
| 220 |
break
|
| 221 |
|
| 222 |
-
success = True
|
| 223 |
-
|
| 224 |
except KeyboardInterrupt:
|
| 225 |
-
|
| 226 |
-
_dbg("[DBG] Interrupted by user")
|
| 227 |
except Exception as exc:
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
|
| 236 |
if __name__ == "__main__":
|
| 237 |
run_inference()
|
|
|
|
| 1 |
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import sys
|
| 5 |
import json
|
| 6 |
import re
|
| 7 |
+
import datetime
|
| 8 |
import traceback
|
| 9 |
import time
|
| 10 |
from typing import List
|
|
|
|
| 27 |
|
| 28 |
from models import AuditorAction
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
class LLMResponse(BaseModel):
|
| 31 |
reasoning: str
|
| 32 |
decisions: List[int]
|
|
|
|
| 39 |
raise ValueError("CRITICAL: HF_TOKEN environment variable is missing.")
|
| 40 |
|
| 41 |
TASK_ID: str = os.getenv("TASK_ID", "anomaly_detection_hard")
|
|
|
|
| 42 |
|
|
|
|
| 43 |
if "easy" in TASK_ID.lower():
|
| 44 |
_DEFAULT_MAX = 5
|
| 45 |
elif "medium" in TASK_ID.lower():
|
|
|
|
| 71 |
{"reasoning": "Trade 1 has high risk. Trade 2 is safe.", "decisions": [1, 0, 1]}
|
| 72 |
"""
|
| 73 |
|
| 74 |
+
def _ts() -> str:
|
| 75 |
+
return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
| 76 |
+
|
| 77 |
def _build_user_prompt(step: int, features: list[list[float]]) -> str:
|
| 78 |
lines = [
|
| 79 |
f"Step {step}: You have {len(features)} flagged trades to audit.",
|
|
|
|
| 153 |
content = response.choices[0].message.content or ""
|
| 154 |
return _parse_llm_decisions(content, len(features))
|
| 155 |
except Exception as e:
|
|
|
|
| 156 |
time.sleep(1)
|
| 157 |
|
|
|
|
| 158 |
fallback_decisions = []
|
| 159 |
for row in features:
|
| 160 |
if len(row) >= 4:
|
|
|
|
| 165 |
return fallback_decisions
|
| 166 |
|
| 167 |
def run_inference() -> None:
|
| 168 |
+
episode_id: str = "unknown"
|
| 169 |
+
total_reward: float = 0.0
|
| 170 |
steps_completed: int = 0
|
| 171 |
+
status: str = "SUCCESS"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
try:
|
| 174 |
env = FinAuditorEnvironment()
|
| 175 |
obs = env.reset()
|
| 176 |
+
episode_id = getattr(env.state, 'episode_id', "test_run")
|
| 177 |
|
| 178 |
+
start_payload = {
|
| 179 |
+
"episode_id": episode_id,
|
| 180 |
+
"model": MODEL_NAME,
|
| 181 |
+
"difficulty": TASK_ID,
|
| 182 |
+
"max_steps": MAX_STEPS
|
| 183 |
+
}
|
| 184 |
+
print(f"[START] {json.dumps(start_payload)}", flush=True)
|
| 185 |
|
| 186 |
for step_num in range(1, MAX_STEPS + 1):
|
| 187 |
+
step_reward = 0.0
|
| 188 |
features = obs.features
|
| 189 |
|
| 190 |
if not features:
|
| 191 |
action = AuditorAction(decisions=[])
|
| 192 |
+
_last_reasoning = "Empty matrix."
|
| 193 |
else:
|
| 194 |
decisions = _call_llm(step_num, features)
|
| 195 |
action = AuditorAction(decisions=decisions)
|
| 196 |
|
| 197 |
obs = env.step(action)
|
| 198 |
+
# Apply safe floor fallback in inference just in case
|
| 199 |
+
step_reward = obs.reward if obs.reward is not None else 0.01
|
| 200 |
+
total_reward += step_reward
|
| 201 |
steps_completed = step_num
|
| 202 |
|
| 203 |
+
# FIX: Used round(..., 4) to prevent collapsing small fractions into 0.00
|
| 204 |
+
step_payload = {
|
| 205 |
+
"step": step_num,
|
| 206 |
+
"anomalies": len(features),
|
| 207 |
+
"reward": round(float(step_reward), 4),
|
| 208 |
+
"cumulative_reward": round(float(total_reward), 4),
|
| 209 |
+
"done": bool(obs.done),
|
| 210 |
+
"error": None,
|
| 211 |
+
"reasoning": _last_reasoning[:120].replace('\n', ' ') + "...",
|
| 212 |
+
"tp": getattr(env.state, 'last_tp', 0),
|
| 213 |
+
"tn": getattr(env.state, 'last_tn', 0),
|
| 214 |
+
"fp": getattr(env.state, 'last_fp', 0),
|
| 215 |
+
"fn": getattr(env.state, 'last_fn', 0)
|
| 216 |
+
}
|
| 217 |
+
print(f"[STEP] {json.dumps(step_payload)}", flush=True)
|
| 218 |
|
| 219 |
if obs.done:
|
| 220 |
break
|
| 221 |
|
|
|
|
|
|
|
| 222 |
except KeyboardInterrupt:
|
| 223 |
+
status = "INTERRUPTED"
|
|
|
|
| 224 |
except Exception as exc:
|
| 225 |
+
status = "ERROR"
|
| 226 |
+
traceback.print_exc(file=sys.stderr)
|
| 227 |
+
|
| 228 |
+
avg_reward = total_reward / max(steps_completed, 1)
|
| 229 |
+
|
| 230 |
+
# FIX: Used round(..., 4) for terminal payload outputs as well
|
| 231 |
+
end_payload = {
|
| 232 |
+
"total_reward": round(float(total_reward), 4),
|
| 233 |
+
"avg_reward": round(float(avg_reward), 4),
|
| 234 |
+
"status": status
|
| 235 |
+
}
|
| 236 |
+
print(f"[END] {json.dumps(end_payload)}", flush=True)
|
| 237 |
|
| 238 |
if __name__ == "__main__":
|
| 239 |
run_inference()
|
server/app.py
CHANGED
|
@@ -44,9 +44,6 @@ except ImportError as e:
|
|
| 44 |
# PHASE 2: SYSTEM STATE & AUTHORITY TRACKING
|
| 45 |
# ==============================================================================
|
| 46 |
|
| 47 |
-
# FIX: Global pointer to capture the OpenEnv-managed instance
|
| 48 |
-
active_env_instance = None
|
| 49 |
-
|
| 50 |
if HAS_ENV and NATIVE_VERIFIED:
|
| 51 |
class TrackedFinAuditorEnvironment(FinAuditorEnvironment):
|
| 52 |
"""Wrapper class to capture the environment instance created by OpenEnv"""
|
|
@@ -56,7 +53,8 @@ if HAS_ENV and NATIVE_VERIFIED:
|
|
| 56 |
active_env_instance = self
|
| 57 |
|
| 58 |
# OpenEnv creates the FastAPI app and instantiates TrackedFinAuditorEnvironment internally
|
| 59 |
-
|
|
|
|
| 60 |
else:
|
| 61 |
app = FastAPI(title="PayGorn (MOCK MODE)")
|
| 62 |
@app.post("/reset")
|
|
|
|
| 44 |
# PHASE 2: SYSTEM STATE & AUTHORITY TRACKING
|
| 45 |
# ==============================================================================
|
| 46 |
|
|
|
|
|
|
|
|
|
|
| 47 |
if HAS_ENV and NATIVE_VERIFIED:
|
| 48 |
class TrackedFinAuditorEnvironment(FinAuditorEnvironment):
|
| 49 |
"""Wrapper class to capture the environment instance created by OpenEnv"""
|
|
|
|
| 53 |
active_env_instance = self
|
| 54 |
|
| 55 |
# OpenEnv creates the FastAPI app and instantiates TrackedFinAuditorEnvironment internally
|
| 56 |
+
active_env_instance = FinAuditorEnvironment()
|
| 57 |
+
app = create_app(lambda: active_env_instance, AuditorAction, AuditorObservation)
|
| 58 |
else:
|
| 59 |
app = FastAPI(title="PayGorn (MOCK MODE)")
|
| 60 |
@app.post("/reset")
|
server/fin_auditor_environment.py
CHANGED
|
@@ -82,10 +82,8 @@ class FinAuditorEnvironment(Environment):
|
|
| 82 |
self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
|
| 83 |
self.sim_time_ns = 0
|
| 84 |
|
| 85 |
-
# 1. READ TASK_ID FROM ENVIRONMENT
|
| 86 |
task_id = os.getenv("TASK_ID", "anomaly_detection_hard").lower()
|
| 87 |
|
| 88 |
-
# 2. MAP TO C++ DIFFICULTY ENUM & SYNC YAML STEPS
|
| 89 |
if "easy" in task_id:
|
| 90 |
self.difficulty = hft_auditor.Difficulty.EASY
|
| 91 |
self._MAX_EPISODE_STEPS = 5
|
|
@@ -98,60 +96,42 @@ class FinAuditorEnvironment(Environment):
|
|
| 98 |
|
| 99 |
def reset(self) -> AuditorObservation:
|
| 100 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 101 |
-
|
| 102 |
-
# Re-initialize the engine for a clean episode
|
| 103 |
-
self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
|
| 104 |
-
self.sim_time_ns = 0
|
| 105 |
-
|
| 106 |
-
# Generate the first batch so step 1 has data to evaluate
|
| 107 |
-
self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
|
| 108 |
-
|
| 109 |
-
# Advance time past Δ_max to expire the batch
|
| 110 |
-
self.sim_time_ns += 6_000_000_000
|
| 111 |
self.engine.tick(self.sim_time_ns)
|
| 112 |
|
| 113 |
-
# Get the anomaly matrix for the agent (features for step 1)
|
| 114 |
-
anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
|
| 115 |
-
|
| 116 |
return FinAuditorObservation(
|
| 117 |
-
features=
|
| 118 |
-
message=
|
| 119 |
-
reward=0.
|
| 120 |
done=False
|
| 121 |
)
|
| 122 |
|
| 123 |
def step(self, action: AuditorAction) -> AuditorObservation: # type: ignore[override]
|
| 124 |
self._state.step_count += 1
|
| 125 |
|
| 126 |
-
#
|
| 127 |
-
|
| 128 |
if action and action.decisions:
|
| 129 |
action_array = np.array(action.decisions, dtype=np.uint8)
|
| 130 |
raw_reward = float(self.engine.compute_reward(action_array))
|
| 131 |
-
|
| 132 |
-
# Map raw reward bounds [-4.0, 40.0] to a [0, 1] percentage
|
| 133 |
normalized_raw = (raw_reward + 4.0) / 44.0
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
# 2. GENERATE NEW DATA (Using procedural C++ engine)
|
| 142 |
self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
|
| 143 |
|
| 144 |
-
# 3. ADVANCE TIME & EXPIRE
|
| 145 |
self.sim_time_ns += 6_000_000_000
|
| 146 |
self.engine.tick(self.sim_time_ns)
|
| 147 |
|
| 148 |
-
# 4. EXTRACT NEW MATRIX
|
| 149 |
anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
|
| 150 |
total_anomalies = len(anomalies)
|
| 151 |
|
| 152 |
done = self._state.step_count >= self._MAX_EPISODE_STEPS
|
| 153 |
|
| 154 |
-
# Expose C++ tracking metrics to the Python state
|
| 155 |
self._state.last_tp = self.engine.last_tp
|
| 156 |
self._state.last_tn = self.engine.last_tn
|
| 157 |
self._state.last_fp = self.engine.last_fp
|
|
|
|
| 82 |
self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
|
| 83 |
self.sim_time_ns = 0
|
| 84 |
|
|
|
|
| 85 |
task_id = os.getenv("TASK_ID", "anomaly_detection_hard").lower()
|
| 86 |
|
|
|
|
| 87 |
if "easy" in task_id:
|
| 88 |
self.difficulty = hft_auditor.Difficulty.EASY
|
| 89 |
self._MAX_EPISODE_STEPS = 5
|
|
|
|
| 96 |
|
| 97 |
def reset(self) -> AuditorObservation:
|
| 98 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 99 |
+
self.sim_time_ns += self._DELTA_MAX_NS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
self.engine.tick(self.sim_time_ns)
|
| 101 |
|
|
|
|
|
|
|
|
|
|
| 102 |
return FinAuditorObservation(
|
| 103 |
+
features=[],
|
| 104 |
+
message="Fin Auditor engine ready.",
|
| 105 |
+
reward=0.01, # Safe minimum floor, not divided
|
| 106 |
done=False
|
| 107 |
)
|
| 108 |
|
| 109 |
def step(self, action: AuditorAction) -> AuditorObservation: # type: ignore[override]
|
| 110 |
self._state.step_count += 1
|
| 111 |
|
| 112 |
+
# FIX: OpenEnv grader:reward evaluates EACH step's reward independently.
|
| 113 |
+
# Must be strictly in (0.01, 0.99) for every step, no exceptions.
|
| 114 |
if action and action.decisions:
|
| 115 |
action_array = np.array(action.decisions, dtype=np.uint8)
|
| 116 |
raw_reward = float(self.engine.compute_reward(action_array))
|
| 117 |
+
# Map raw bounds [-4.0, 40.0] -> [0.0, 1.0]
|
|
|
|
| 118 |
normalized_raw = (raw_reward + 4.0) / 44.0
|
| 119 |
+
# Clamp strictly inside (0.01, 0.999)
|
| 120 |
+
step_reward = max(0.01, min(0.99, normalized_raw))
|
| 121 |
+
else:
|
| 122 |
+
# Empty decisions (no-op step) - return safe floor, NOT 0.0
|
| 123 |
+
step_reward = 0.01
|
| 124 |
+
|
|
|
|
|
|
|
| 125 |
self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
|
| 126 |
|
|
|
|
| 127 |
self.sim_time_ns += 6_000_000_000
|
| 128 |
self.engine.tick(self.sim_time_ns)
|
| 129 |
|
|
|
|
| 130 |
anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
|
| 131 |
total_anomalies = len(anomalies)
|
| 132 |
|
| 133 |
done = self._state.step_count >= self._MAX_EPISODE_STEPS
|
| 134 |
|
|
|
|
| 135 |
self._state.last_tp = self.engine.last_tp
|
| 136 |
self._state.last_tn = self.engine.last_tn
|
| 137 |
self._state.last_fp = self.engine.last_fp
|