""" Evaluation API endpoint for LLM Deployment System. This module implements the evaluation API that receives repository submissions from students and queues them for evaluation. """ import json import time from datetime import datetime, timezone from typing import Dict, Any, Optional import threading import queue import requests from flask import Flask, request, jsonify, Response from flask_cors import CORS from utils.config import config from .database import db_manager, TaskStatus from utils.logger import get_logger, log_request_info, log_error, log_evaluation from utils.github_utils import get_github_manager, GitHubUtils logger = get_logger(__name__) class EvaluationAPI: """Evaluation API server class.""" def __init__(self): self.app = Flask(__name__) self.app.config['SECRET_KEY'] = config.SECRET_KEY self.app.config['MAX_CONTENT_LENGTH'] = config.MAX_CONTENT_LENGTH # Enable CORS CORS(self.app, origins=config.CORS_ORIGINS) # Setup evaluation queue self.evaluation_queue = queue.Queue() self.worker_thread = None # Setup routes self._setup_routes() # Initialize managers self.github_manager = None if config.ENABLE_GITHUB_INTEGRATION: try: self.github_manager = get_github_manager() except Exception as e: logger.warning(f"Failed to initialize GitHub manager: {e}") # Start evaluation worker self.start_worker() def _setup_routes(self): """Setup API routes.""" @self.app.route('/health', methods=['GET']) def health_check(): """Health check endpoint.""" return jsonify({ 'status': 'healthy', 'timestamp': datetime.now(timezone.utc).isoformat(), 'queue_size': self.evaluation_queue.qsize(), 'version': '1.0.0' }) @self.app.route('/evaluate', methods=['POST']) def handle_evaluation_submission(): """Handle repository evaluation submissions from students.""" start_time = time.time() try: data = request.get_json() if not data: return jsonify({'error': 'No JSON data provided'}), 400 # Validate required fields required_fields = ['email', 'task', 'round', 'nonce', 'repo_url', 'commit_sha'] for field in required_fields: if field not in data: return jsonify({'error': f'Missing required field: {field}'}), 400 # Validate task exists and nonce matches task = db_manager.get_task_by_id(data['task']) if not task: return jsonify({'error': 'Invalid task ID'}), 400 if task.nonce != data['nonce']: return jsonify({'error': 'Invalid nonce'}), 400 if task.status not in [TaskStatus.SENT, TaskStatus.RECEIVED]: return jsonify({'error': 'Task not in correct state'}), 400 # Update task status db_manager.update_task_status(data['task'], TaskStatus.RECEIVED) # Store repository information repo_data = { 'repo_url': data['repo_url'], 'commit_sha': data['commit_sha'], 'pages_url': data.get('pages_url') } repo = db_manager.create_repository(task.id, repo_data) # Queue for evaluation evaluation_job = { 'repository_id': repo.id, 'task_id': data['task'], 'repo_url': data['repo_url'], 'commit_sha': data['commit_sha'], 'pages_url': data.get('pages_url'), 'submitted_at': datetime.now(timezone.utc).isoformat() } self.evaluation_queue.put(evaluation_job) logger.info(f"Queued evaluation for task {data['task']}, repository {repo.id}") log_evaluation(logger, data['task'], 'queued', repository_id=repo.id) log_request_info(logger, 'POST', '/evaluate', 200, time.time() - start_time) return jsonify({ 'status': 'received', 'repository_id': repo.id, 'message': 'Repository submission received and queued for evaluation' }), 200 except Exception as e: logger.error(f"Error handling evaluation submission: {e}") log_error(logger, e, 'handle_evaluation_submission') return jsonify({'error': 'Internal server error'}), 500 @self.app.route('/evaluate/status/', methods=['GET']) def get_evaluation_status(task_id: str): """Get evaluation status for a task.""" try: task = db_manager.get_task_by_id(task_id) if not task: return jsonify({'error': 'Task not found'}), 404 # Get repository and evaluations repos = task.repos if not repos: return jsonify({'status': 'no_submission'}) latest_repo = max(repos, key=lambda r: r.submitted_at) evaluations = db_manager.get_evaluations_by_repository(latest_repo.id) completed_evaluations = [e for e in evaluations if e.status.value in ['passed', 'failed']] if not evaluations: status = 'queued' elif len(completed_evaluations) == len(evaluations): status = 'completed' else: status = 'evaluating' return jsonify({ 'task_id': task_id, 'status': status, 'total_checks': len(evaluations), 'completed_checks': len(completed_evaluations), 'evaluations': [ { 'check_name': e.check_name, 'status': e.status.value, 'score': e.score, 'reason': e.reason } for e in evaluations ] }) except Exception as e: logger.error(f"Error getting evaluation status for {task_id}: {e}") return jsonify({'error': 'Internal server error'}), 500 @self.app.route('/evaluate/results/', methods=['GET']) def get_evaluation_results(task_id: str): """Get detailed evaluation results for a task.""" try: task = db_manager.get_task_by_id(task_id) if not task: return jsonify({'error': 'Task not found'}), 404 repos = task.repos if not repos: return jsonify({'error': 'No repositories submitted'}), 404 latest_repo = max(repos, key=lambda r: r.submitted_at) evaluations = db_manager.get_evaluations_by_repository(latest_repo.id) results = [] for evaluation in evaluations: results.append({ 'check_name': evaluation.check_name, 'status': evaluation.status.value, 'score': evaluation.score, 'reason': evaluation.reason, 'logs': evaluation.logs, 'evaluated_at': evaluation.evaluated_at.isoformat() if evaluation.evaluated_at else None, 'duration_seconds': evaluation.duration_seconds }) return jsonify({ 'task_id': task_id, 'repository_url': latest_repo.repo_url, 'commit_sha': latest_repo.commit_sha, 'pages_url': latest_repo.pages_url, 'results': results }) except Exception as e: logger.error(f"Error getting evaluation results for {task_id}: {e}") return jsonify({'error': 'Internal server error'}), 500 @self.app.errorhandler(404) def not_found(error): return jsonify({'error': 'Not found'}), 404 @self.app.errorhandler(405) def method_not_allowed(error): return jsonify({'error': 'Method not allowed'}), 405 @self.app.errorhandler(413) def file_too_large(error): return jsonify({'error': 'File too large'}), 413 @self.app.errorhandler(500) def internal_error(error): logger.error(f"Internal server error: {error}") return jsonify({'error': 'Internal server error'}), 500 def start_worker(self): """Start the evaluation worker thread.""" if self.worker_thread is None or not self.worker_thread.is_alive(): self.worker_thread = threading.Thread(target=self._evaluation_worker, daemon=True) self.worker_thread.start() logger.info("Started evaluation worker thread") def _evaluation_worker(self): """Background worker to process evaluation jobs.""" logger.info("Evaluation worker started") while True: try: # Get job from queue (blocking) job = self.evaluation_queue.get(timeout=1) logger.info(f"Processing evaluation job for repository {job['repository_id']}") log_evaluation(logger, job['task_id'], 'processing', repository_id=job['repository_id']) # Process the evaluation self._process_evaluation(job) # Mark job as done self.evaluation_queue.task_done() except queue.Empty: # No jobs in queue, continue continue except Exception as e: logger.error(f"Error in evaluation worker: {e}") log_error(logger, e, '_evaluation_worker') time.sleep(5) # Wait before retrying def _process_evaluation(self, job: Dict[str, Any]): """Process a single evaluation job.""" try: from .evaluate import evaluate_repository repository_id = job['repository_id'] repo_url = job['repo_url'] commit_sha = job['commit_sha'] pages_url = job['pages_url'] # Run evaluation results = evaluate_repository( repo_url=repo_url, commit_sha=commit_sha, pages_url=pages_url ) # Store results in database for result in results: db_manager.add_evaluation(repository_id, result) logger.info(f"Completed evaluation for repository {repository_id}") log_evaluation(logger, job['task_id'], 'completed', repository_id=repository_id) except Exception as e: logger.error(f"Failed to process evaluation for repository {job['repository_id']}: {e}") log_error(logger, e, f"_process_evaluation_{job['repository_id']}") # Store error result error_result = { 'check_name': 'system_error', 'status': 'error', 'score': 0.0, 'reason': f'Evaluation failed: {str(e)}', 'logs': {'error': str(e)} } db_manager.add_evaluation(job['repository_id'], error_result) # Global evaluation API instance evaluation_api = EvaluationAPI() def get_evaluation_api() -> EvaluationAPI: """Get the global evaluation API instance.""" return evaluation_api def create_evaluation_app() -> Flask: """Create Flask application instance for evaluation API.""" return evaluation_api.app if __name__ == '__main__': evaluation_api.app.run(host=config.API_HOST, port=config.API_PORT + 1, debug=config.FLASK_DEBUG)