#!/usr/bin/env python3 """ ATLAS提交处理 - OSS模式 使用阿里云OSS替代git/http提交方式 """ import json import os import sys from datetime import datetime from typing import Dict, Any, Tuple from pathlib import Path # 导入OSS提交处理器 try: from src.oss.oss_submission_handler import OSSSubmissionHandler OSS_AVAILABLE = True except ImportError as e: print(f"⚠️ OSS module not available, using fallback mode: {e}") OSS_AVAILABLE = False def format_error(msg): return f"

{msg}

" def format_success(msg): return f"

{msg}

" def format_warning(msg): return f"

{msg}

" def validate_sage_submission(submission_data: Dict[str, Any]) -> Tuple[bool, str]: """验证ATLAS基准提交格式""" # 检查必需的顶级字段 required_fields = ["submission_org", "submission_email", "predictions"] for field in required_fields: if field not in submission_data: return False, f"Missing required field: {field}" # 验证邮箱格式(基本验证) email = submission_data["submission_email"] if "@" not in email or "." not in email: return False, "Invalid email format" # 验证predictions predictions = submission_data["predictions"] if not isinstance(predictions, list) or len(predictions) == 0: return False, "predictions must be a non-empty list" for i, prediction in enumerate(predictions): # 检查必需的prediction字段 pred_required_fields = ["original_question_id", "content", "reasoning_content"] for field in pred_required_fields: if field not in prediction: return False, f"Missing field in prediction {i}: {field}" # 验证content数组 content = prediction["content"] reasoning_content = prediction["reasoning_content"] if not isinstance(content, list) or len(content) != 4: return False, f"content in prediction {i} must be a list with 4 items" if not isinstance(reasoning_content, list): return False, f"reasoning_content in prediction {i} must be a list" # # reasoning_content可以为空列表,或者包含4个项目 # if len(reasoning_content) != 0 and len(reasoning_content) != 4: # return False, f"reasoning_content in prediction {i} must be an empty list or contain 4 items" # 验证question ID if not isinstance(prediction["original_question_id"], int): return False, f"question ID in prediction {i} must be an integer" return True, "Submission format is valid" def save_submission_file(submission_data: Dict[str, Any], submissions_dir: str = "./submissions") -> str: """保存提交文件到指定目录""" # 确保submissions目录存在 os.makedirs(submissions_dir, exist_ok=True) # 生成文件名 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") # 获取模型名和组织名 model_name = submission_data.get("model_name", "UnknownModel") model_name = model_name.replace(" ", "_").replace("/", "_").replace("\\", "_").replace("-", "_") org_name = submission_data["submission_org"].replace(" ", "_").replace("/", "_").replace("\\", "_") # 格式: submission_模型名_组织_时间戳.json filename = f"submission_{model_name}_{org_name}_{timestamp}.json" # 完整文件路径 file_path = os.path.join(submissions_dir, filename) # 保存文件 with open(file_path, 'w', encoding='utf-8') as f: json.dump(submission_data, f, indent=2, ensure_ascii=False) return file_path def process_sage_submission_simple(submission_file, model_name=None, org_name=None, email=None) -> str: """ 处理ATLAS基准提交文件 - 文件收集模式 只负责验证和保存,不进行评测 """ try: # 读取提交的文件 if submission_file is None: return format_error("❌ No file uploaded. Please select a JSON file.") # submission_file是文件路径字符串 try: with open(submission_file, 'r', encoding='utf-8') as f: content = f.read() except Exception as e: return format_error(f"❌ Error reading file: {str(e)}") # 解析JSON try: submission_data = json.loads(content) except json.JSONDecodeError as e: return format_error(f"❌ Invalid JSON format: {str(e)}") # 如果表单提供了模型名、组织名和邮箱,使用表单数据 if model_name: submission_data["model_name"] = model_name.strip() if org_name and email: submission_data["submission_org"] = org_name.strip() submission_data["submission_email"] = email.strip() # 验证提交格式 is_valid, message = validate_sage_submission(submission_data) if not is_valid: return format_error(f"❌ Submission validation failed: {message}") # 保存提交文件 try: saved_path = save_submission_file(submission_data) print(f"✅ Submission file saved to: {saved_path}") # OSS上传策略 if OSS_AVAILABLE: try: # 使用OSS提交处理器 oss_handler = OSSSubmissionHandler() result = oss_handler.process_sage_submission(submission_data, org_name, email) # 如果OSS处理成功,直接返回结果 if "Submission successful" in result or "successful" in result.lower(): return result else: # OSS失败,继续使用备用模式 print(f"⚠️ OSS submission failed, using fallback mode: {result}") except Exception as e: print(f"⚠️ OSS submission exception, using fallback mode: {e}") # 备用模式:本地保存 filename = os.path.basename(saved_path) # 生成成功消息 org = submission_data["submission_org"] email_addr = submission_data["submission_email"] num_predictions = len(submission_data["predictions"]) success_msg = format_success(f""" 🎉 Submission successful!

📋 Submission Information:
• Organization: {org}
• Email: {email_addr}
• Number of predictions: {num_predictions} questions
• Filename: {filename}

🧪 Thank you for participating in the ATLAS scientific reasoning benchmark! """) return success_msg except Exception as e: return format_error(f"❌ Error saving submission file: {str(e)}") except Exception as e: return format_error(f"❌ Submission processing failed: {str(e)}") def get_submission_stats(submissions_dir: str = "./submissions") -> Dict[str, Any]: """获取提交统计信息""" if not os.path.exists(submissions_dir): return {"total": 0, "recent": []} submissions = [] for filename in os.listdir(submissions_dir): if filename.startswith("submission_") and filename.endswith(".json"): file_path = os.path.join(submissions_dir, filename) try: with open(file_path, 'r', encoding='utf-8') as f: data = json.load(f) # 提取信息 timestamp_str = filename.split("_")[-1].replace(".json", "") try: timestamp = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S") formatted_time = timestamp.strftime("%Y-%m-%d %H:%M") except: formatted_time = timestamp_str submissions.append({ "org": data.get("submission_org", "Unknown"), "email": data.get("submission_email", ""), "time": formatted_time, "predictions": len(data.get("predictions", [])) }) except Exception: continue # 按时间排序,最新的在前 submissions.sort(key=lambda x: x["time"], reverse=True) return { "total": len(submissions), "recent": submissions[:10] # 最近10个 } # 移除了原有的HTTP推送函数,现在使用OSS模式