Spaces:
Running on Zero
Running on Zero
Update evaluator.py
Browse files- evaluator.py +18 -1
evaluator.py
CHANGED
|
@@ -199,7 +199,15 @@ def generate_executive_summary(reports: list) -> dict:
|
|
| 199 |
'errors_detected': sum(len(r['failures']) for r in reports)
|
| 200 |
}
|
| 201 |
|
| 202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
if isinstance(input_data, str):
|
| 204 |
with open(input_data, 'r', encoding='utf-8') as f:
|
| 205 |
sessions_data = pd.DataFrame(json.load(f))
|
|
@@ -208,6 +216,15 @@ def evaluate_trajectories_batch(input_data: Union[str, List[Dict[str, Any]]], pr
|
|
| 208 |
else:
|
| 209 |
raise ValueError("Invalid input format.")
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
active_profile = profile or DEFAULT_STRICTNESS_PROFILE
|
| 212 |
profile_cfg = PROFILES_CONFIG.get(active_profile.lower(), PROFILES_CONFIG["standard"])
|
| 213 |
max_tool_latency = profile_cfg["max_latency_ms"]
|
|
|
|
| 199 |
'errors_detected': sum(len(r['failures']) for r in reports)
|
| 200 |
}
|
| 201 |
|
| 202 |
+
MAX_FREE_STEPS = 20
|
| 203 |
+
|
| 204 |
+
def evaluate_trajectories_batch(
|
| 205 |
+
input_data: Union[str, List[Dict[str, Any]]],
|
| 206 |
+
profile: str = None,
|
| 207 |
+
run_stress_test: bool = False,
|
| 208 |
+
plan: str = "free"
|
| 209 |
+
) -> dict:
|
| 210 |
+
|
| 211 |
if isinstance(input_data, str):
|
| 212 |
with open(input_data, 'r', encoding='utf-8') as f:
|
| 213 |
sessions_data = pd.DataFrame(json.load(f))
|
|
|
|
| 216 |
else:
|
| 217 |
raise ValueError("Invalid input format.")
|
| 218 |
|
| 219 |
+
if plan == "free":
|
| 220 |
+
for _, session in sessions_data.iterrows():
|
| 221 |
+
node_count = len(session.get("nodes", []))
|
| 222 |
+
if node_count > MAX_FREE_STEPS:
|
| 223 |
+
return {
|
| 224 |
+
"error": f"Free Tier Limit Exceeded: Session [{session.get('session_id', 'unknown')}] has {node_count} steps (Max allowed on Free: {MAX_FREE_STEPS}). Upgrade to Pro for unlimited trajectory depth.",
|
| 225 |
+
"status_code": 429
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
active_profile = profile or DEFAULT_STRICTNESS_PROFILE
|
| 229 |
profile_cfg = PROFILES_CONFIG.get(active_profile.lower(), PROFILES_CONFIG["standard"])
|
| 230 |
max_tool_latency = profile_cfg["max_latency_ms"]
|