Buckets:
tahamajs/Sysmem2_in_AI / ComputerAssignments /CA17_reinforcement_learning_math /scripts /evaluate.py
| #!/usr/bin/env python3 | |
| import argparse | |
| import os | |
| import sys | |
| import json | |
| from pathlib import Path | |
| sys.path.append(str(Path(__file__).parent.parent / "src")) | |
| from src.evaluation.evaluator import ModelEvaluator | |
| from src.agents.math_agent import MathReasoningAgent | |
| from src.config import Config | |
| from src.utils.logging_utils import setup_logging | |
| from loguru import logger | |
| def parse_args(): | |
| parser = argparse.ArgumentParser(description="Evaluate RL-LLM models") | |
| parser.add_argument( | |
| "--model_path", type=str, required=True, help="Path to trained model" | |
| ) | |
| parser.add_argument( | |
| "--config", type=str, default="config.yaml", help="Path to configuration file" | |
| ) | |
| parser.add_argument( | |
| "--num_episodes", type=int, default=100, help="Number of evaluation episodes" | |
| ) | |
| parser.add_argument( | |
| "--difficulty_levels", | |
| nargs="+", | |
| default=["easy", "medium", "hard"], | |
| help="Difficulty levels to evaluate", | |
| ) | |
| parser.add_argument( | |
| "--problem_types", | |
| nargs="+", | |
| default=["arithmetic", "algebra", "geometry", "calculus"], | |
| help="Problem types to evaluate", | |
| ) | |
| parser.add_argument( | |
| "--output_dir", | |
| type=str, | |
| default="./evaluation_results", | |
| help="Output directory for evaluation results", | |
| ) | |
| parser.add_argument( | |
| "--log_level", | |
| type=str, | |
| default="INFO", | |
| choices=["DEBUG", "INFO", "WARNING", "ERROR"], | |
| help="Logging level", | |
| ) | |
| parser.add_argument( | |
| "--device", type=str, default="auto", help="Device to use (cuda, cpu, auto)" | |
| ) | |
| parser.add_argument( | |
| "--compare_difficulties", | |
| action="store_true", | |
| help="Compare performance across difficulty levels", | |
| ) | |
| parser.add_argument( | |
| "--compare_types", | |
| action="store_true", | |
| help="Compare performance across problem types", | |
| ) | |
| parser.add_argument( | |
| "--statistical_analysis", | |
| action="store_true", | |
| help="Perform statistical analysis on results", | |
| ) | |
| return parser.parse_args() | |
| def load_model(model_path: str, config: Config): | |
| logger.info(f"Loading model from {model_path}") | |
| agent = MathReasoningAgent(config.model) | |
| if os.path.exists(model_path): | |
| try: | |
| import torch | |
| checkpoint = torch.load(model_path, map_location="cpu") | |
| if "model_state_dict" in checkpoint: | |
| agent.model.load_state_dict(checkpoint["model_state_dict"]) | |
| logger.info("Loaded model state dict") | |
| else: | |
| logger.warning("No model state dict found in checkpoint") | |
| if "few_shot_examples" in checkpoint: | |
| agent.few_shot_examples = checkpoint["few_shot_examples"] | |
| logger.info(f"Loaded {len(agent.few_shot_examples)} few-shot examples") | |
| except Exception as e: | |
| logger.warning(f"Failed to load checkpoint: {e}") | |
| logger.info("Using untrained model") | |
| else: | |
| logger.warning(f"Model path {model_path} does not exist, using untrained model") | |
| return agent | |
| def main(): | |
| args = parse_args() | |
| log_file = os.path.join(args.output_dir, "evaluation.log") | |
| os.makedirs(args.output_dir, exist_ok=True) | |
| setup_logging(log_file, args.log_level) | |
| logger.info("Starting RL-LLM evaluation") | |
| logger.info(f"Arguments: {args}") | |
| try: | |
| if os.path.exists(args.config): | |
| config = Config.from_yaml(args.config) | |
| logger.info(f"Loaded configuration from {args.config}") | |
| else: | |
| logger.warning( | |
| f"Configuration file {args.config} not found, using defaults" | |
| ) | |
| config = Config() | |
| config.model.device = args.device | |
| agent = load_model(args.model_path, config) | |
| evaluator = ModelEvaluator(config, args.output_dir) | |
| logger.info("Running main evaluation") | |
| result = evaluator.evaluate_agent( | |
| agent=agent, | |
| num_episodes=args.num_episodes, | |
| difficulty_levels=args.difficulty_levels, | |
| problem_types=args.problem_types, | |
| save_results=True, | |
| ) | |
| logger.info("Evaluation completed") | |
| logger.info(f"Average reward: {result.avg_reward:.3f}") | |
| logger.info(f"Success rate: {result.success_rate:.3f}") | |
| logger.info(f"Total episodes: {result.num_episodes}") | |
| if args.compare_difficulties: | |
| logger.info("Running difficulty comparison") | |
| difficulty_results = evaluator.evaluate_by_difficulty( | |
| agent, episodes_per_difficulty=args.num_episodes // 3 | |
| ) | |
| logger.info("Difficulty comparison results:") | |
| for difficulty, diff_result in difficulty_results.items(): | |
| logger.info( | |
| f"{difficulty}: avg_reward={diff_result.avg_reward:.3f}, " | |
| f"success_rate={diff_result.success_rate:.3f}" | |
| ) | |
| if args.compare_types: | |
| logger.info("Running problem type comparison") | |
| type_results = evaluator.evaluate_by_problem_type( | |
| agent, episodes_per_type=args.num_episodes // 4 | |
| ) | |
| logger.info("Problem type comparison results:") | |
| for problem_type, type_result in type_results.items(): | |
| logger.info( | |
| f"{problem_type}: avg_reward={type_result.avg_reward:.3f}, " | |
| f"success_rate={type_result.success_rate:.3f}" | |
| ) | |
| if args.statistical_analysis: | |
| logger.info("Performing statistical analysis") | |
| from src.evaluation.statistical_analysis import StatisticalAnalyzer | |
| analyzer = StatisticalAnalyzer() | |
| rewards = [ep["reward"] for ep in result.episode_results] | |
| normality_result = analyzer.perform_normality_test(rewards) | |
| logger.info( | |
| f"Normality test: is_normal={normality_result.get('is_normal', False)}" | |
| ) | |
| ci_result = analyzer.calculate_confidence_interval(rewards) | |
| logger.info( | |
| f"95% CI: [{ci_result['lower_bound']:.3f}, {ci_result['upper_bound']:.3f}]" | |
| ) | |
| stats_file = os.path.join(args.output_dir, "statistical_analysis.json") | |
| with open(stats_file, "w") as f: | |
| json.dump( | |
| { | |
| "normality_test": normality_result, | |
| "confidence_interval": ci_result, | |
| "descriptive_stats": { | |
| "mean": result.avg_reward, | |
| "std": result.std_reward, | |
| "min": result.min_reward, | |
| "max": result.max_reward, | |
| "median": result.median_reward, | |
| }, | |
| }, | |
| f, | |
| indent=2, | |
| default=str, | |
| ) | |
| logger.info(f"Statistical analysis saved to {stats_file}") | |
| logger.info("=" * 50) | |
| logger.info("EVALUATION SUMMARY") | |
| logger.info("=" * 50) | |
| logger.info(f"Model: {args.model_path}") | |
| logger.info(f"Episodes: {result.num_episodes}") | |
| logger.info( | |
| f"Average Reward: {result.avg_reward:.3f} ± {result.std_reward:.3f}" | |
| ) | |
| logger.info(f"Success Rate: {result.success_rate:.3f}") | |
| logger.info(f"Min Reward: {result.min_reward:.3f}") | |
| logger.info(f"Max Reward: {result.max_reward:.3f}") | |
| logger.info(f"Median Reward: {result.median_reward:.3f}") | |
| logger.info(f"Evaluation Time: {result.evaluation_time:.2f}s") | |
| logger.info(f"Results saved to: {args.output_dir}") | |
| except Exception as e: | |
| logger.error(f"Evaluation failed: {e}") | |
| raise | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 7.98 kB
- Xet hash:
- 10a245e152d3d4a663fb240302598890a5c57368217cb3725aff43d58a4f936f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.