chore: remove MTEB results analysis script

This commit removes the script for analyzing MTEB results. The script was responsible for processing benchmark results, categorizing tasks, calculating averages, and updating the README.md file with a results table. The script is no longer needed.

analyze_mteb_results.py

@@ -1,311 +0,0 @@
-#!/usr/bin/env python
-"""
-MTEB Results Analysis Script.
-
-This script analyzes MTEB benchmark results from the results directory,
-categorizes tasks, calculates averages, and updates the README.md with
-a comprehensive results table.
-"""
-
-import json
-import re
-from pathlib import Path
-
-# Task category mappings based on MTEB benchmark structure
-TASK_CATEGORIES = {
-	# Classification tasks
-	"AmazonCounterfactualClassification": "Classification",
-	"AmazonReviewsClassification": "Classification",
-	"Banking77Classification": "Classification",
-	"EmotionClassification": "Classification",
-	"ImdbClassification": "Classification",
-	"MassiveIntentClassification": "Classification",
-	"MassiveScenarioClassification": "Classification",
-	"MTOPDomainClassification": "Classification",
-	"MTOPIntentClassification": "Classification",
-	"ToxicConversationsClassification": "Classification",
-	"TweetSentimentExtractionClassification": "Classification",
-	# Clustering tasks
-	"ArxivClusteringP2P": "Clustering",
-	"ArxivClusteringS2S": "Clustering",
-	"BiorxivClusteringP2P": "Clustering",
-	"BiorxivClusteringS2S": "Clustering",
-	"MedrxivClusteringP2P": "Clustering",
-	"MedrxivClusteringS2S": "Clustering",
-	"RedditClustering": "Clustering",
-	"RedditClusteringP2P": "Clustering",
-	"StackExchangeClustering": "Clustering",
-	"StackExchangeClusteringP2P": "Clustering",
-	"TwentyNewsgroupsClustering": "Clustering",
-	# Pair Classification tasks
-	"SprintDuplicateQuestions": "PairClassification",
-	"TwitterSemEval2015": "PairClassification",
-	"TwitterURLCorpus": "PairClassification",
-	# Reranking tasks
-	"AskUbuntuDupQuestions": "Reranking",
-	"MindSmallReranking": "Reranking",
-	"SciDocsRR": "Reranking",
-	"StackOverflowDupQuestions": "Reranking",
-	# Retrieval tasks
-	"ArguAna": "Retrieval",
-	"ClimateFEVER": "Retrieval",
-	"CQADupstackRetrieval": "Retrieval",
-	"DBPedia": "Retrieval",
-	"FEVER": "Retrieval",
-	"FiQA2018": "Retrieval",
-	"HotpotQA": "Retrieval",
-	"MSMARCO": "Retrieval",
-	"NFCorpus": "Retrieval",
-	"NQ": "Retrieval",
-	"QuoraRetrieval": "Retrieval",
-	"SCIDOCS": "Retrieval",
-	"SciFact": "Retrieval",
-	"Touche2020": "Retrieval",
-	"TRECCOVID": "Retrieval",
-	# Code retrieval tasks
-	"CodeSearchNetCCRetrieval": "CodeRetrieval",
-	"COIRCodeSearchNetRetrieval": "CodeRetrieval",
-	"StackOverflowQA": "CodeRetrieval",
-	"AppsRetrieval": "CodeRetrieval",
-	"CodeTransOceanContest": "CodeRetrieval",
-	"CodeTransOceanDL": "CodeRetrieval",
-	"CodeFeedbackMT": "CodeRetrieval",
-	"SyntheticText2SQL": "CodeRetrieval",
-	"CosQA": "CodeRetrieval",
-	# STS (Semantic Textual Similarity) tasks
-	"BIOSSES": "STS",
-	"SICK-R": "STS",
-	"STS12": "STS",
-	"STS13": "STS",
-	"STS14": "STS",
-	"STS15": "STS",
-	"STS16": "STS",
-	"STS17": "STS",
-	"STS22": "STS",
-	"STSBenchmark": "STS",
-	"SummEval": "STS",
-}
-
-
-def load_mteb_results(results_dir: Path) -> dict[str, dict]:
-	"""Load all MTEB results from the results directory."""
-	results = {}
-
-	for json_file in results_dir.glob("*.json"):
-		if json_file.name == "model_meta.json":
-			continue
-
-		try:
-			with json_file.open() as f:
-				data = json.load(f)
-				task_name = data.get("task_name", json_file.stem)
-				results[task_name] = data
-		except (json.JSONDecodeError, KeyError):
-			pass
-
-	return results
-
-
-def extract_main_score(result_data: dict) -> float:
-	"""Extract the main score from a task result."""
-	try:
-		scores = result_data["scores"]["test"][0]
-		return scores["main_score"]
-	except (KeyError, IndexError, TypeError):
-		return 0.0
-
-
-def categorize_tasks(results: dict[str, dict]) -> dict[str, list[tuple[str, float]]]:
-	"""Categorize tasks and extract their scores."""
-	categories: dict[str, list[tuple[str, float]]] = {}
-
-	for task_name, result_data in results.items():
-		# Get category from mapping, or try to infer from task name
-		category = TASK_CATEGORIES.get(task_name)
-
-		if not category:
-			# Try to infer category from task name patterns
-			if "Classification" in task_name:
-				category = "Classification"
-			elif "Clustering" in task_name:
-				category = "Clustering"
-			elif "Retrieval" in task_name or "QA" in task_name:
-				category = "Retrieval"
-			elif "STS" in task_name or "SICK" in task_name or "BIOSSES" in task_name:
-				category = "STS"
-			elif "Code" in task_name or "SQL" in task_name:
-				category = "CodeRetrieval"
-			else:
-				category = "Other"
-
-		score = extract_main_score(result_data)
-
-		if category not in categories:
-			categories[category] = []
-		categories[category].append((task_name, score))
-
-	# Sort tasks within each category
-	for category_tasks in categories.values():
-		category_tasks.sort(key=lambda x: x[0])
-
-	return categories
-
-
-def calculate_averages(categories: dict[str, list[tuple[str, float]]]) -> dict[str, float]:
-	"""Calculate average scores for each category."""
-	averages = {}
-
-	for category, tasks in categories.items():
-		scores = [score for _, score in tasks if score > 0]  # Exclude failed tasks (score = 0)
-		if scores:
-			averages[category] = sum(scores) / len(scores)
-		else:
-			averages[category] = 0.0
-
-	return averages
-
-
-def generate_results_table(categories: dict[str, list[tuple[str, float]]], averages: dict[str, float]) -> str:
-	"""Generate a markdown table with the results."""
-	# Calculate overall average
-	all_scores = []
-	for tasks in categories.values():
-		all_scores.extend([score for _, score in tasks if score > 0])
-
-	overall_avg = sum(all_scores) / len(all_scores) if all_scores else 0.0
-
-	# Create table
-	table_lines = [
-		"## MTEB Benchmark Results",
-		"",
-		f"**Overall Average Score: {overall_avg:.4f}**",
-		"",
-		"| Category | Task | Score |",
-		"|----------|------|-------|",
-	]
-
-	# Sort categories for consistent ordering
-	category_order = [
-		"Classification",
-		"Clustering",
-		"PairClassification",
-		"Reranking",
-		"Retrieval",
-		"CodeRetrieval",
-		"STS",
-		"Other",
-	]
-
-	for category in category_order:
-		if category not in categories:
-			continue
-
-		tasks = categories[category]
-		if not tasks:
-			continue
-
-		# Add category average row
-		avg_score = averages[category]
-		table_lines.append(f"| **{category}** | **Average** | **{avg_score:.4f}** |")
-
-		# Add individual tasks
-		for task_name, score in tasks:
-			if score > 0:  # Only show successful tasks
-				table_lines.append(f"| | {task_name} | {score:.4f} |")
-			else:
-				table_lines.append(f"| | {task_name} | Failed |")
-
-		table_lines.append("| | | |")  # Empty row for spacing
-
-	# Add summary statistics
-	table_lines.extend(
-		[
-			"",
-			"### Summary Statistics",
-			"",
-			f"- **Total Tasks**: {sum(len(tasks) for tasks in categories.values())}",
-			f"- **Successful Tasks**: {len(all_scores)}",
-			f"- **Failed Tasks**: {sum(len(tasks) for tasks in categories.values()) - len(all_scores)}",
-			f"- **Overall Average**: {overall_avg:.4f}",
-			"",
-			"### Category Averages",
-			"",
-		]
-	)
-
-	for category in category_order:
-		if category in averages and categories.get(category):
-			avg = averages[category]
-			task_count = len([s for _, s in categories[category] if s > 0])
-			table_lines.append(f"- **{category}**: {avg:.4f} ({task_count} tasks)")
-
-	return "\n".join(table_lines)
-
-
-def update_readme(results_table: str, readme_path: Path = Path("README.md")) -> None:
-	"""Update the README.md file with the results table."""
-	if not readme_path.exists():
-		return
-
-	# Read current README
-	with readme_path.open() as f:
-		content = f.read()
-
-	# Find the insertion point or replace existing MTEB results
-	mteb_pattern = r"## MTEB Benchmark Results.*?(?=\n## |\n# |\Z)"
-
-	if re.search(mteb_pattern, content, re.DOTALL):
-		# Replace existing MTEB results section
-		new_content = re.sub(mteb_pattern, results_table, content, flags=re.DOTALL)
-	# Find a good insertion point (before Acknowledgments section or at the end)
-	elif "## Acknowledgments" in content:
-		new_content = content.replace("## Acknowledgments", f"{results_table}\n\n## Acknowledgments")
-	elif "## License" in content:
-		new_content = content.replace("## License", f"{results_table}\n\n## License")
-	else:
-		# Add at the end
-		new_content = f"{content}\n\n{results_table}"
-
-	# Write updated README
-	with readme_path.open("w") as f:
-		f.write(new_content)
-
-
-def main() -> None:
-	"""Main function to analyze MTEB results and update README."""
-	results_dir = Path("mteb_results/gte-Qwen2-7B-instruct-M2V-Distilled/distilled")
-
-	if not results_dir.exists():
-		return
-
-	results = load_mteb_results(results_dir)
-
-	if not results:
-		return
-
-	categories = categorize_tasks(results)
-
-	averages = calculate_averages(categories)
-
-	results_table = generate_results_table(categories, averages)
-
-	update_readme(results_table)
-
-	# Print summary to console
-
-	sum(len(tasks) for tasks in categories.values())
-	successful_tasks = sum(len([s for _, s in tasks if s > 0]) for tasks in categories.values())
-
-	if successful_tasks > 0:
-		all_scores = []
-		for tasks in categories.values():
-			all_scores.extend([score for _, score in tasks if score > 0])
-		sum(all_scores) / len(all_scores)
-
-	for category, tasks in categories.items():
-		len([s for _, s in tasks if s > 0])
-		averages.get(category, 0.0)
-
-
-if __name__ == "__main__":
-	main()