sdawdsdw commited on
Commit
07a5dfd
·
verified ·
1 Parent(s): d75ec6b

Update evaluator.py

Browse files
Files changed (1) hide show
  1. evaluator.py +18 -1
evaluator.py CHANGED
@@ -199,7 +199,15 @@ def generate_executive_summary(reports: list) -> dict:
199
  'errors_detected': sum(len(r['failures']) for r in reports)
200
  }
201
 
202
- def evaluate_trajectories_batch(input_data: Union[str, List[Dict[str, Any]]], profile: str = None, run_stress_test: bool = False) -> dict:
 
 
 
 
 
 
 
 
203
  if isinstance(input_data, str):
204
  with open(input_data, 'r', encoding='utf-8') as f:
205
  sessions_data = pd.DataFrame(json.load(f))
@@ -208,6 +216,15 @@ def evaluate_trajectories_batch(input_data: Union[str, List[Dict[str, Any]]], pr
208
  else:
209
  raise ValueError("Invalid input format.")
210
 
 
 
 
 
 
 
 
 
 
211
  active_profile = profile or DEFAULT_STRICTNESS_PROFILE
212
  profile_cfg = PROFILES_CONFIG.get(active_profile.lower(), PROFILES_CONFIG["standard"])
213
  max_tool_latency = profile_cfg["max_latency_ms"]
 
199
  'errors_detected': sum(len(r['failures']) for r in reports)
200
  }
201
 
202
+ MAX_FREE_STEPS = 20
203
+
204
+ def evaluate_trajectories_batch(
205
+ input_data: Union[str, List[Dict[str, Any]]],
206
+ profile: str = None,
207
+ run_stress_test: bool = False,
208
+ plan: str = "free"
209
+ ) -> dict:
210
+
211
  if isinstance(input_data, str):
212
  with open(input_data, 'r', encoding='utf-8') as f:
213
  sessions_data = pd.DataFrame(json.load(f))
 
216
  else:
217
  raise ValueError("Invalid input format.")
218
 
219
+ if plan == "free":
220
+ for _, session in sessions_data.iterrows():
221
+ node_count = len(session.get("nodes", []))
222
+ if node_count > MAX_FREE_STEPS:
223
+ return {
224
+ "error": f"Free Tier Limit Exceeded: Session [{session.get('session_id', 'unknown')}] has {node_count} steps (Max allowed on Free: {MAX_FREE_STEPS}). Upgrade to Pro for unlimited trajectory depth.",
225
+ "status_code": 429
226
+ }
227
+
228
  active_profile = profile or DEFAULT_STRICTNESS_PROFILE
229
  profile_cfg = PROFILES_CONFIG.get(active_profile.lower(), PROFILES_CONFIG["standard"])
230
  max_tool_latency = profile_cfg["max_latency_ms"]