Buckets:
| from typing import Dict, List, Any, Optional | |
| from dataclasses import dataclass | |
| from datetime import datetime | |
| class EvaluationResult: | |
| agent_name: str | |
| task: str | |
| success: bool | |
| execution_time: float | |
| reasoning_quality: float | |
| tool_usage_efficiency: float | |
| error_count: int | |
| timestamp: datetime | |
| def to_dict(self) -> Dict[str, Any]: | |
| return { | |
| "agent_name": self.agent_name, | |
| "task": self.task, | |
| "success": self.success, | |
| "execution_time": self.execution_time, | |
| "reasoning_quality": self.reasoning_quality, | |
| "tool_usage_efficiency": self.tool_usage_efficiency, | |
| "error_count": self.error_count, | |
| "timestamp": self.timestamp.isoformat(), | |
| } | |
| class AgentEvaluator: | |
| def __init__(self): | |
| self.evaluation_history = [] | |
| def evaluate_agent( | |
| self, agent_result: Any, task: str, agent_name: str | |
| ) -> EvaluationResult: | |
| success = getattr(agent_result, "success", False) | |
| execution_time = getattr(agent_result, "total_execution_time", 0.0) | |
| reasoning_quality = getattr(agent_result, "reasoning_quality", 0.0) | |
| error_count = len(getattr(agent_result, "error_messages", [])) | |
| tool_usage_stats = getattr(agent_result, "tool_usage_stats", {}) | |
| total_tool_calls = sum(tool_usage_stats.values()) | |
| tool_usage_efficiency = ( | |
| min(1.0, total_tool_calls / 10.0) if total_tool_calls > 0 else 0.0 | |
| ) | |
| result = EvaluationResult( | |
| agent_name=agent_name, | |
| task=task, | |
| success=success, | |
| execution_time=execution_time, | |
| reasoning_quality=reasoning_quality, | |
| tool_usage_efficiency=tool_usage_efficiency, | |
| error_count=error_count, | |
| timestamp=datetime.now(), | |
| ) | |
| self.evaluation_history.append(result) | |
| return result | |
| def get_performance_summary(self) -> Dict[str, Any]: | |
| if not self.evaluation_history: | |
| return { | |
| "total_evaluations": 0, | |
| "success_rate": 0.0, | |
| "avg_execution_time": 0.0, | |
| "avg_reasoning_quality": 0.0, | |
| } | |
| total_evaluations = len(self.evaluation_history) | |
| successful_evaluations = sum(1 for r in self.evaluation_history if r.success) | |
| success_rate = successful_evaluations / total_evaluations | |
| avg_execution_time = ( | |
| sum(r.execution_time for r in self.evaluation_history) / total_evaluations | |
| ) | |
| avg_reasoning_quality = ( | |
| sum(r.reasoning_quality for r in self.evaluation_history) | |
| / total_evaluations | |
| ) | |
| return { | |
| "total_evaluations": total_evaluations, | |
| "success_rate": success_rate, | |
| "avg_execution_time": avg_execution_time, | |
| "avg_reasoning_quality": avg_reasoning_quality, | |
| "evaluations": [r.to_dict() for r in self.evaluation_history], | |
| } | |
Xet Storage Details
- Size:
- 3.04 kB
- Xet hash:
- ad1731aa3028f55ab4a0b73147bb770591ba09721752476df84b8cbccf64c15b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.