Remove multi-swe-bench from OpenHands Index

- Remove multi-swe-bench from benchmark mappings in simple_data_loader.py
- Remove multi-swe-bench from generate_mock_jsonl.py
- Update FRONTEND_DEVELOPMENT_DESCRIPTION to reference SWE-bench Multimodal (not Multi-SWE-bench)
- Remove multi-swe-bench from about.py benchmark lists and acknowledgements
- Update DATA_STRUCTURE.md to remove multi-swe-bench references
- Keep 5 categories (Frontend uses swe-bench-multimodal, not multi-swe-bench)

DATA_STRUCTURE.md

@@ -12,7 +12,6 @@ openhands-index-results/
 │   ├── test.jsonl            # Test split results
 │   ├── validation.jsonl      # Validation split results
 │   ├── swe-bench.jsonl       # Individual benchmark results
-│   ├── multi-swe-bench.jsonl
 │   ├── swe-bench-multimodal.jsonl
 │   ├── swt-bench.jsonl
 │   ├── commit0.jsonl
@@ -58,10 +57,6 @@ The configuration file defines the benchmark structure:
             "name": "swe-bench",
             "tags": ["swe-bench"]
           },
-          {
-            "name": "multi-swe-bench",
-            "tags": ["multi-swe-bench"]
-          },
           {
             "name": "swe-bench-multimodal",
             "tags": ["swe-bench-multimodal"]
@@ -87,10 +82,6 @@ The configuration file defines the benchmark structure:
             "name": "swe-bench",
             "tags": ["swe-bench"]
           },
-          {
-            "name": "multi-swe-bench",
-            "tags": ["multi-swe-bench"]
-          },
           {
             "name": "swe-bench-multimodal",
             "tags": ["swe-bench-multimodal"]

about.py

@@ -34,11 +34,10 @@ def build_page():
             """
             <h2>What Does OpenHands Index Include?</h2>
             <p>
-                OpenHands Index aggregates results from 6 key benchmarks for evaluating AI coding agents:
+                OpenHands Index aggregates results from 5 key benchmarks for evaluating AI coding agents:
             </p>
             <ul class="info-list">
                 <li><strong>SWE-bench</strong>: Repository-level bug fixing from real GitHub issues</li>
-                <li><strong>Multi-SWE-bench</strong>: Multi-repository software engineering tasks</li>
                 <li><strong>SWE-bench Multimodal</strong>: Bug fixing with visual context</li>
                 <li><strong>SWT-bench</strong>: Web development and testing tasks</li>
                 <li><strong>Commit0</strong>: Commit message generation and code understanding</li>
@@ -116,7 +115,7 @@ def build_page():
                 To submit your agent's evaluation results to the OpenHands Index:
             </p>
             <ol class="info-list">
-                <li>Run your agent on the supported benchmarks (SWE-bench, Multi-SWE-bench, SWE-bench Multimodal, SWT-bench, Commit0, GAIA)</li>
+                <li>Run your agent on the supported benchmarks (SWE-bench, SWE-bench Multimodal, SWT-bench, Commit0, GAIA)</li>
                 <li>Format your results according to the data structure documented in the repository</li>
                 <li>Submit a pull request to <a href="https://github.com/OpenHands/openhands-index-results" target="_blank" class="primary-link-button">github.com/OpenHands/openhands-index-results</a></li>
                 <li>Your submission should include:
@@ -164,7 +163,6 @@ def build_page():
             </p>
             <ul class="info-list">
                 <li><a href="https://www.swebench.com/" target="_blank">SWE-bench</a> - Princeton NLP Group</li>
-                <li><a href="https://github.com/multi-swe-bench/multi-swe-bench" target="_blank">Multi-SWE-bench</a> - Multi-SWE-bench Team</li>
                 <li><a href="https://github.com/OpenHands/SWE-bench-multimodal" target="_blank">SWE-bench Multimodal</a> - OpenHands Team</li>
                 <li><a href="https://github.com/logic-star-ai/swt-bench" target="_blank">SWT-bench</a> - Logic Star AI</li>
                 <li><a href="https://github.com/commit-0/commit0" target="_blank">Commit0</a> - Commit0 Team</li>

content.py

@@ -57,7 +57,7 @@ For detailed results, use the links above to explore individual benchmark pages.
 FRONTEND_DEVELOPMENT_DESCRIPTION = """
 The **Frontend** category evaluates agents on their ability to build user interfaces and web applications. This tests skills in HTML, CSS, JavaScript frameworks, responsive design, and creating interactive user experiences.
 <br><br>
-This category includes Multi-SWE-bench, which challenges agents to work across multiple repositories and coordinate changes in complex web application architectures.
+This category includes SWE-bench Multimodal, which challenges agents to solve GitHub issues that include visual context like screenshots and diagrams.
 <br>
 """
 TEST_GENERATION_DESCRIPTION = """

generate_mock_jsonl.py

@@ -4,18 +4,13 @@ import os
 from pathlib import Path
 from datetime import datetime
 
-# Define the 6 benchmarks
+# Define the 5 benchmarks
 BENCHMARKS = {
     "swe-bench": {
         "tags": ["swe-bench"],
         "metric": "resolve_rate",
         "metric_display": "Resolve Rate (%)"
     },
-    "multi-swe-bench": {
-        "tags": ["multi-swe-bench"],
-        "metric": "resolve_rate",
-        "metric_display": "Resolve Rate (%)"
-    },
     "swe-bench-multimodal": {
         "tags": ["swe-bench-multimodal"],
         "metric": "resolve_rate",
@@ -46,7 +41,6 @@ MOCK_AGENTS = [
         "openness": "closed",
         "scores": {
             "swe-bench": 48.3,
-            "multi-swe-bench": 35.2,
             "swe-bench-multimodal": 42.1,
             "swt-bench": 65.4,
             "commit0": 71.2,
@@ -59,7 +53,6 @@ MOCK_AGENTS = [
         "openness": "closed",
         "scores": {
             "swe-bench": 45.1,
-            "multi-swe-bench": 32.8,
             "swe-bench-multimodal": 39.5,
             "swt-bench": 62.3,
             "commit0": 68.9,
@@ -72,7 +65,6 @@ MOCK_AGENTS = [
         "openness": "closed",
         "scores": {
             "swe-bench": 38.7,
-            "multi-swe-bench": 28.4,
             "swe-bench-multimodal": 34.2,
             "swt-bench": 54.1,
             "commit0": 61.5,
@@ -85,7 +77,6 @@ MOCK_AGENTS = [
         "openness": "closed",
         "scores": {
             "swe-bench": 32.5,
-            "multi-swe-bench": 24.1,
             "swe-bench-multimodal": 28.9,
             "swt-bench": 47.8,
             "commit0": 55.3,
@@ -98,7 +89,6 @@ MOCK_AGENTS = [
         "openness": "closed",
         "scores": {
             "swe-bench": 29.8,
-            "multi-swe-bench": 21.5,
             "swe-bench-multimodal": 25.7,
             "swt-bench": 44.2,
             "commit0": 52.1,

simple_data_loader.py

@@ -147,7 +147,6 @@ class SimpleLeaderboardViewer:
                 'swe-bench': ['Issue Resolution'],
                 'swe-bench-multimodal': ['Frontend'],
                 'commit0': ['Greenfield'],
-                'multi-swe-bench': ['Issue Resolution'],
                 'swt-bench': ['Testing'],
                 'gaia': ['Information Gathering'],
             }