SamaKool commited on
Commit
a7ca8f0
·
2 Parent(s): 44762af8b64f33

fix: pinned nanobind, disabled benchmark for docker, and isolated state endpoint

Browse files
Files changed (3) hide show
  1. inference.py +48 -46
  2. server/app.py +2 -4
  3. server/fin_auditor_environment.py +13 -33
inference.py CHANGED
@@ -1,19 +1,10 @@
1
  #!/usr/bin/env python3
2
- # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
3
- # inference.py — OpenEnv Evaluation Script (Hackathon Submission)
4
- #
5
- # STDOUT FORMAT (strict regex compliance):
6
- # [START] task=<task_name> env=<benchmark> model=<model_name>
7
- # [STEP] step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
8
- # [END] success=<true|false> steps=<n> rewards=<r1,r2,...,rn>
9
- #
10
- # ALL debug output goes to stderr. NO JSON on stdout. NO extra whitespace.
11
- # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
12
 
13
  import os
14
  import sys
15
  import json
16
  import re
 
17
  import traceback
18
  import time
19
  from typing import List
@@ -36,10 +27,6 @@ except ImportError:
36
 
37
  from models import AuditorAction
38
 
39
- # ── Debug logger: ONLY to stderr ─────────────────────────────────────────────
40
- def _dbg(msg: str) -> None:
41
- print(msg, file=sys.stderr, flush=True)
42
-
43
  class LLMResponse(BaseModel):
44
  reasoning: str
45
  decisions: List[int]
@@ -52,9 +39,7 @@ if not HF_TOKEN:
52
  raise ValueError("CRITICAL: HF_TOKEN environment variable is missing.")
53
 
54
  TASK_ID: str = os.getenv("TASK_ID", "anomaly_detection_hard")
55
- ENV_NAME: str = "fin_auditor"
56
 
57
- # FIX: Sync the inference max_steps default with the active task
58
  if "easy" in TASK_ID.lower():
59
  _DEFAULT_MAX = 5
60
  elif "medium" in TASK_ID.lower():
@@ -86,6 +71,9 @@ Example:
86
  {"reasoning": "Trade 1 has high risk. Trade 2 is safe.", "decisions": [1, 0, 1]}
87
  """
88
 
 
 
 
89
  def _build_user_prompt(step: int, features: list[list[float]]) -> str:
90
  lines = [
91
  f"Step {step}: You have {len(features)} flagged trades to audit.",
@@ -165,10 +153,8 @@ def _call_llm(step: int, features: list[list[float]]) -> list[int]:
165
  content = response.choices[0].message.content or ""
166
  return _parse_llm_decisions(content, len(features))
167
  except Exception as e:
168
- _dbg(f"[LLM RETRY {attempt+1}/{max_retries}] {e}")
169
  time.sleep(1)
170
 
171
- _dbg("[LLM] All retries exhausted, using risk_score fallback")
172
  fallback_decisions = []
173
  for row in features:
174
  if len(row) >= 4:
@@ -179,59 +165,75 @@ def _call_llm(step: int, features: list[list[float]]) -> list[int]:
179
  return fallback_decisions
180
 
181
  def run_inference() -> None:
 
 
182
  steps_completed: int = 0
183
- all_rewards: list[float] = []
184
- success: bool = False
185
- error_msg: str | None = None
186
-
187
- # ── [START] — always emitted ──────────────────────────────────────────
188
- print(f"[START] task={TASK_ID} env={ENV_NAME} model={MODEL_NAME}", flush=True)
189
 
190
  try:
191
  env = FinAuditorEnvironment()
192
  obs = env.reset()
 
193
 
194
- _dbg(f"[DBG] Episode started. Features: {len(obs.features)} rows, difficulty: {TASK_ID}")
 
 
 
 
 
 
195
 
196
  for step_num in range(1, MAX_STEPS + 1):
197
- step_reward = 0.0
198
  features = obs.features
199
 
200
  if not features:
201
  action = AuditorAction(decisions=[])
202
- _dbg(f"[DBG] Step {step_num}: Empty feature matrix")
203
  else:
204
  decisions = _call_llm(step_num, features)
205
  action = AuditorAction(decisions=decisions)
206
 
207
  obs = env.step(action)
208
- step_reward = obs.reward if obs.reward is not None else 0.0
209
- all_rewards.append(step_reward)
 
210
  steps_completed = step_num
211
 
212
- # ── [STEP] plain text, 2 decimal places, lowercase bools ────
213
- action_str = ",".join(str(d) for d in action.decisions) if action.decisions else "none"
214
- done_str = "true" if obs.done else "false"
215
- print(f"[STEP] step={step_num} action={action_str} reward={step_reward:.2f} done={done_str} error=null", flush=True)
216
-
217
- _dbg(f"[DBG] Step {step_num}: reward={step_reward:.4f}, features={len(obs.features)}, done={obs.done}")
 
 
 
 
 
 
 
 
 
218
 
219
  if obs.done:
220
  break
221
 
222
- success = True
223
-
224
  except KeyboardInterrupt:
225
- error_msg = "interrupted"
226
- _dbg("[DBG] Interrupted by user")
227
  except Exception as exc:
228
- error_msg = str(exc).replace("\n", " ")[:80]
229
- _dbg(f"[ERROR] {traceback.format_exc()}")
230
- finally:
231
- # ── [END] ALWAYS emitted, even on crash ────────────────────────
232
- success_str = "true" if success else "false"
233
- rewards_str = ",".join(f"{r:.2f}" for r in all_rewards) if all_rewards else "0.00"
234
- print(f"[END] success={success_str} steps={steps_completed} rewards={rewards_str}", flush=True)
 
 
 
 
 
235
 
236
  if __name__ == "__main__":
237
  run_inference()
 
1
  #!/usr/bin/env python3
 
 
 
 
 
 
 
 
 
 
2
 
3
  import os
4
  import sys
5
  import json
6
  import re
7
+ import datetime
8
  import traceback
9
  import time
10
  from typing import List
 
27
 
28
  from models import AuditorAction
29
 
 
 
 
 
30
  class LLMResponse(BaseModel):
31
  reasoning: str
32
  decisions: List[int]
 
39
  raise ValueError("CRITICAL: HF_TOKEN environment variable is missing.")
40
 
41
  TASK_ID: str = os.getenv("TASK_ID", "anomaly_detection_hard")
 
42
 
 
43
  if "easy" in TASK_ID.lower():
44
  _DEFAULT_MAX = 5
45
  elif "medium" in TASK_ID.lower():
 
71
  {"reasoning": "Trade 1 has high risk. Trade 2 is safe.", "decisions": [1, 0, 1]}
72
  """
73
 
74
+ def _ts() -> str:
75
+ return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
76
+
77
  def _build_user_prompt(step: int, features: list[list[float]]) -> str:
78
  lines = [
79
  f"Step {step}: You have {len(features)} flagged trades to audit.",
 
153
  content = response.choices[0].message.content or ""
154
  return _parse_llm_decisions(content, len(features))
155
  except Exception as e:
 
156
  time.sleep(1)
157
 
 
158
  fallback_decisions = []
159
  for row in features:
160
  if len(row) >= 4:
 
165
  return fallback_decisions
166
 
167
  def run_inference() -> None:
168
+ episode_id: str = "unknown"
169
+ total_reward: float = 0.0
170
  steps_completed: int = 0
171
+ status: str = "SUCCESS"
 
 
 
 
 
172
 
173
  try:
174
  env = FinAuditorEnvironment()
175
  obs = env.reset()
176
+ episode_id = getattr(env.state, 'episode_id', "test_run")
177
 
178
+ start_payload = {
179
+ "episode_id": episode_id,
180
+ "model": MODEL_NAME,
181
+ "difficulty": TASK_ID,
182
+ "max_steps": MAX_STEPS
183
+ }
184
+ print(f"[START] {json.dumps(start_payload)}", flush=True)
185
 
186
  for step_num in range(1, MAX_STEPS + 1):
187
+ step_reward = 0.0
188
  features = obs.features
189
 
190
  if not features:
191
  action = AuditorAction(decisions=[])
192
+ _last_reasoning = "Empty matrix."
193
  else:
194
  decisions = _call_llm(step_num, features)
195
  action = AuditorAction(decisions=decisions)
196
 
197
  obs = env.step(action)
198
+ # Apply safe floor fallback in inference just in case
199
+ step_reward = obs.reward if obs.reward is not None else 0.01
200
+ total_reward += step_reward
201
  steps_completed = step_num
202
 
203
+ # FIX: Used round(..., 4) to prevent collapsing small fractions into 0.00
204
+ step_payload = {
205
+ "step": step_num,
206
+ "anomalies": len(features),
207
+ "reward": round(float(step_reward), 4),
208
+ "cumulative_reward": round(float(total_reward), 4),
209
+ "done": bool(obs.done),
210
+ "error": None,
211
+ "reasoning": _last_reasoning[:120].replace('\n', ' ') + "...",
212
+ "tp": getattr(env.state, 'last_tp', 0),
213
+ "tn": getattr(env.state, 'last_tn', 0),
214
+ "fp": getattr(env.state, 'last_fp', 0),
215
+ "fn": getattr(env.state, 'last_fn', 0)
216
+ }
217
+ print(f"[STEP] {json.dumps(step_payload)}", flush=True)
218
 
219
  if obs.done:
220
  break
221
 
 
 
222
  except KeyboardInterrupt:
223
+ status = "INTERRUPTED"
 
224
  except Exception as exc:
225
+ status = "ERROR"
226
+ traceback.print_exc(file=sys.stderr)
227
+
228
+ avg_reward = total_reward / max(steps_completed, 1)
229
+
230
+ # FIX: Used round(..., 4) for terminal payload outputs as well
231
+ end_payload = {
232
+ "total_reward": round(float(total_reward), 4),
233
+ "avg_reward": round(float(avg_reward), 4),
234
+ "status": status
235
+ }
236
+ print(f"[END] {json.dumps(end_payload)}", flush=True)
237
 
238
  if __name__ == "__main__":
239
  run_inference()
server/app.py CHANGED
@@ -44,9 +44,6 @@ except ImportError as e:
44
  # PHASE 2: SYSTEM STATE & AUTHORITY TRACKING
45
  # ==============================================================================
46
 
47
- # FIX: Global pointer to capture the OpenEnv-managed instance
48
- active_env_instance = None
49
-
50
  if HAS_ENV and NATIVE_VERIFIED:
51
  class TrackedFinAuditorEnvironment(FinAuditorEnvironment):
52
  """Wrapper class to capture the environment instance created by OpenEnv"""
@@ -56,7 +53,8 @@ if HAS_ENV and NATIVE_VERIFIED:
56
  active_env_instance = self
57
 
58
  # OpenEnv creates the FastAPI app and instantiates TrackedFinAuditorEnvironment internally
59
- app = create_app(TrackedFinAuditorEnvironment, AuditorAction, AuditorObservation)
 
60
  else:
61
  app = FastAPI(title="PayGorn (MOCK MODE)")
62
  @app.post("/reset")
 
44
  # PHASE 2: SYSTEM STATE & AUTHORITY TRACKING
45
  # ==============================================================================
46
 
 
 
 
47
  if HAS_ENV and NATIVE_VERIFIED:
48
  class TrackedFinAuditorEnvironment(FinAuditorEnvironment):
49
  """Wrapper class to capture the environment instance created by OpenEnv"""
 
53
  active_env_instance = self
54
 
55
  # OpenEnv creates the FastAPI app and instantiates TrackedFinAuditorEnvironment internally
56
+ active_env_instance = FinAuditorEnvironment()
57
+ app = create_app(lambda: active_env_instance, AuditorAction, AuditorObservation)
58
  else:
59
  app = FastAPI(title="PayGorn (MOCK MODE)")
60
  @app.post("/reset")
server/fin_auditor_environment.py CHANGED
@@ -82,10 +82,8 @@ class FinAuditorEnvironment(Environment):
82
  self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
83
  self.sim_time_ns = 0
84
 
85
- # 1. READ TASK_ID FROM ENVIRONMENT
86
  task_id = os.getenv("TASK_ID", "anomaly_detection_hard").lower()
87
 
88
- # 2. MAP TO C++ DIFFICULTY ENUM & SYNC YAML STEPS
89
  if "easy" in task_id:
90
  self.difficulty = hft_auditor.Difficulty.EASY
91
  self._MAX_EPISODE_STEPS = 5
@@ -98,60 +96,42 @@ class FinAuditorEnvironment(Environment):
98
 
99
  def reset(self) -> AuditorObservation:
100
  self._state = State(episode_id=str(uuid4()), step_count=0)
101
-
102
- # Re-initialize the engine for a clean episode
103
- self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
104
- self.sim_time_ns = 0
105
-
106
- # Generate the first batch so step 1 has data to evaluate
107
- self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
108
-
109
- # Advance time past Δ_max to expire the batch
110
- self.sim_time_ns += 6_000_000_000
111
  self.engine.tick(self.sim_time_ns)
112
 
113
- # Get the anomaly matrix for the agent (features for step 1)
114
- anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
115
-
116
  return FinAuditorObservation(
117
- features=anomalies,
118
- message=f"Engine ready. {len(anomalies)} trades awaiting audit.",
119
- reward=0.001 / self._MAX_EPISODE_STEPS, # Safe fractional minimum
120
  done=False
121
  )
122
 
123
  def step(self, action: AuditorAction) -> AuditorObservation: # type: ignore[override]
124
  self._state.step_count += 1
125
 
126
- # 1. REWARD CALCULATION & MATHEMATICAL NORMALIZATION
127
- step_reward = 0.0
128
  if action and action.decisions:
129
  action_array = np.array(action.decisions, dtype=np.uint8)
130
  raw_reward = float(self.engine.compute_reward(action_array))
131
-
132
- # Map raw reward bounds [-4.0, 40.0] to a [0, 1] percentage
133
  normalized_raw = (raw_reward + 4.0) / 44.0
134
-
135
- # Clamp to prevent EXACT 0.0 or 1.0 boundary hits
136
- safe_clamped = max(0.01, min(0.99, normalized_raw))
137
-
138
- # Distribute over episode length so the SUM is strictly in (0, 1)
139
- step_reward = safe_clamped / self._MAX_EPISODE_STEPS
140
-
141
- # 2. GENERATE NEW DATA (Using procedural C++ engine)
142
  self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
143
 
144
- # 3. ADVANCE TIME & EXPIRE
145
  self.sim_time_ns += 6_000_000_000
146
  self.engine.tick(self.sim_time_ns)
147
 
148
- # 4. EXTRACT NEW MATRIX
149
  anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
150
  total_anomalies = len(anomalies)
151
 
152
  done = self._state.step_count >= self._MAX_EPISODE_STEPS
153
 
154
- # Expose C++ tracking metrics to the Python state
155
  self._state.last_tp = self.engine.last_tp
156
  self._state.last_tn = self.engine.last_tn
157
  self._state.last_fp = self.engine.last_fp
 
82
  self.engine = hft_auditor.ReconciliationEngine(self._RING_BUFFER_CAPACITY)
83
  self.sim_time_ns = 0
84
 
 
85
  task_id = os.getenv("TASK_ID", "anomaly_detection_hard").lower()
86
 
 
87
  if "easy" in task_id:
88
  self.difficulty = hft_auditor.Difficulty.EASY
89
  self._MAX_EPISODE_STEPS = 5
 
96
 
97
  def reset(self) -> AuditorObservation:
98
  self._state = State(episode_id=str(uuid4()), step_count=0)
99
+ self.sim_time_ns += self._DELTA_MAX_NS
 
 
 
 
 
 
 
 
 
100
  self.engine.tick(self.sim_time_ns)
101
 
 
 
 
102
  return FinAuditorObservation(
103
+ features=[],
104
+ message="Fin Auditor engine ready.",
105
+ reward=0.01, # Safe minimum floor, not divided
106
  done=False
107
  )
108
 
109
  def step(self, action: AuditorAction) -> AuditorObservation: # type: ignore[override]
110
  self._state.step_count += 1
111
 
112
+ # FIX: OpenEnv grader:reward evaluates EACH step's reward independently.
113
+ # Must be strictly in (0.01, 0.99) for every step, no exceptions.
114
  if action and action.decisions:
115
  action_array = np.array(action.decisions, dtype=np.uint8)
116
  raw_reward = float(self.engine.compute_reward(action_array))
117
+ # Map raw bounds [-4.0, 40.0] -> [0.0, 1.0]
 
118
  normalized_raw = (raw_reward + 4.0) / 44.0
119
+ # Clamp strictly inside (0.01, 0.999)
120
+ step_reward = max(0.01, min(0.99, normalized_raw))
121
+ else:
122
+ # Empty decisions (no-op step) - return safe floor, NOT 0.0
123
+ step_reward = 0.01
124
+
 
 
125
  self.engine.generate_batch(self.difficulty, self._INGEST_CHUNK_SIZE, self.sim_time_ns)
126
 
 
127
  self.sim_time_ns += 6_000_000_000
128
  self.engine.tick(self.sim_time_ns)
129
 
 
130
  anomalies: list[list[float]] = self.engine.get_anomaly_matrix().tolist()
131
  total_anomalies = len(anomalies)
132
 
133
  done = self._state.step_count >= self._MAX_EPISODE_STEPS
134
 
 
135
  self._state.last_tp = self.engine.last_tp
136
  self._state.last_tn = self.engine.last_tn
137
  self._state.last_fp = self.engine.last_fp