Account for git LFS

Makefile

@@ -11,3 +11,6 @@ quality:
 	python -m black --check --line-length 119 .
 	python -m isort --check-only .
 	ruff check .
+
+test:
+	pytest

app.py

@@ -1,3 +1,4 @@
+import base64
 import io
 from datetime import datetime, timezone
 
@@ -5,7 +6,9 @@ import gradio as gr
 import pandas as pd
 import requests
 
-LEADERBOARD_URL = "https://raw.githubusercontent.com/upgini/mle-bench/main/" "rankings/low/tabular/overall_ranks.csv"
+# GitHub API endpoint for the file (handles Git LFS files)
+LEADERBOARD_API_URL = "https://api.github.com/repos/upgini/mle-bench/contents/rankings/low/tabular/overall_ranks.csv"
+LEADERBOARD_GITHUB_URL = "https://github.com/upgini/mle-bench/blob/main/rankings/low/tabular/overall_ranks.csv"
 
 DISPLAY_COLUMNS = [
     "experiment_id",
@@ -20,11 +23,41 @@ DISPLAY_COLUMNS = [
 
 
 def download_leaderboard() -> pd.DataFrame:
-    """Download the remote leaderboard CSV and return a cleaned dataframe."""
-    response = requests.get(LEADERBOARD_URL, timeout=30)
+    """Download the remote leaderboard CSV from GitHub (handles Git LFS) and return a cleaned dataframe."""
+    # Use GitHub API to get file content (handles Git LFS files)
+    response = requests.get(LEADERBOARD_API_URL, timeout=30)
     response.raise_for_status()
 
-    df = pd.read_csv(io.StringIO(response.text))
+    api_data = response.json()
+
+    # Get file content - GitHub API handles Git LFS files
+    # If content is in the response, decode it; otherwise use download_url
+    if "content" in api_data:
+        # Decode base64 content
+        try:
+            csv_content = base64.b64decode(api_data["content"]).decode("utf-8")
+        except Exception as e:
+            raise ValueError(f"Failed to decode file content: {e}")
+
+        # Check if it's a Git LFS pointer file
+        if csv_content.startswith("version https://git-lfs.github.com/spec/v1"):
+            # For LFS files, use the download_url which points to the actual file
+            download_url = api_data.get("download_url")
+            if not download_url:
+                raise ValueError("Git LFS file found but no download_url available")
+            # Download the actual file content
+            lfs_response = requests.get(download_url, timeout=30)
+            lfs_response.raise_for_status()
+            csv_content = lfs_response.text
+    elif "download_url" in api_data:
+        # Large files don't include content, use download_url directly
+        download_response = requests.get(api_data["download_url"], timeout=30)
+        download_response.raise_for_status()
+        csv_content = download_response.text
+    else:
+        raise ValueError("No content or download_url found in API response")
+
+    df = pd.read_csv(io.StringIO(csv_content))
     missing_cols = [col for col in DISPLAY_COLUMNS if col not in df.columns]
     if missing_cols:
         raise ValueError(f"Leaderboard is missing expected columns: {', '.join(missing_cols)}")
@@ -42,7 +75,7 @@ def refresh_leaderboard():
     """Fetch the leaderboard and build the status message for the UI."""
     df = download_leaderboard()
     status = (
-        f"Showing data from [GitHub]({LEADERBOARD_URL}). "
+        f"Showing data from [GitHub]({LEADERBOARD_GITHUB_URL}). "
         f"Last refreshed: {datetime.now(timezone.utc):%Y-%m-%d %H:%M UTC}."
     )
     return df, status
@@ -71,4 +104,4 @@ with gr.Blocks(title="Upgini MLE-Bench Leaderboard") as demo:
     demo.load(refresh_leaderboard, outputs=[leaderboard_table, status_text])
     refresh_button.click(refresh_leaderboard, outputs=[leaderboard_table, status_text])
 
-demo.queue(concurrency_count=8).launch()
+demo.queue(default_concurrency_limit=8).launch()

pyproject.toml

@@ -11,3 +11,10 @@ line_length = 119
 
 [tool.black]
 line-length = 119
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "-v"

requirements.txt

@@ -9,6 +9,7 @@ huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
+pytest
 requests
 python-dateutil
 tqdm

tests/test_leaderboard.py

@@ -0,0 +1,321 @@
+"""Unit tests for leaderboard functionality."""
+
+import base64
+from unittest.mock import Mock, patch
+
+import pandas as pd
+import pytest
+import requests
+
+from app import DISPLAY_COLUMNS, download_leaderboard, refresh_leaderboard
+
+
+def create_github_api_response(csv_content, is_lfs_pointer=False, use_download_url=False):
+    """Helper to create a mock GitHub API response."""
+    api_response = Mock()
+    api_response.raise_for_status = Mock()
+
+    if use_download_url:
+        # For large files, API doesn't include content, only download_url
+        api_response.json.return_value = {
+            "download_url": "https://github.com/test/file.csv",
+            "sha": "test_sha",
+        }
+        # Second call for download_url
+        download_response = Mock()
+        download_response.text = csv_content
+        download_response.raise_for_status = Mock()
+        return [api_response, download_response]
+    elif is_lfs_pointer:
+        # Create a Git LFS pointer file
+        lfs_pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:test123\nsize 100"
+        encoded_content = base64.b64encode(lfs_pointer.encode("utf-8")).decode("utf-8")
+        api_response.json.return_value = {
+            "content": encoded_content,
+            "download_url": "https://github.com/test/file.csv",
+            "sha": "test_sha",
+        }
+        # Second call for download_url (LFS files need to be downloaded)
+        download_response = Mock()
+        download_response.text = csv_content
+        download_response.raise_for_status = Mock()
+        return [api_response, download_response]
+    else:
+        encoded_content = base64.b64encode(csv_content.encode("utf-8")).decode("utf-8")
+        api_response.json.return_value = {
+            "content": encoded_content,
+            "download_url": "https://github.com/test/file.csv",
+            "sha": "test_sha",
+        }
+        return [api_response]
+
+
+@pytest.fixture
+def sample_csv_data():
+    """Sample CSV data matching the expected leaderboard format."""
+    return """experiment_id,mean_normalized_score,std_normalized_score,mean_medal_pct,sem_medal_pct,Agent,LLM(s) used,Date
+exp_001,0.854321,0.012345,0.876543,0.009876,Agent A,GPT-4,2024-01-15
+exp_002,0.789012,0.023456,0.765432,0.012345,Agent B,Claude-3,2024-01-20
+exp_003,0.912345,0.008765,0.923456,0.007654,Agent C,GPT-4,2024-02-01"""
+
+
+@pytest.fixture
+def sample_csv_with_extra_columns():
+    """Sample CSV with extra columns that should be filtered out."""
+    return """experiment_id,mean_normalized_score,std_normalized_score,mean_medal_pct,sem_medal_pct,Agent,LLM(s) used,Date,extra_col
+exp_001,0.854321,0.012345,0.876543,0.009876,Agent A,GPT-4,2024-01-15,extra_value
+exp_002,0.789012,0.023456,0.765432,0.012345,Agent B,Claude-3,2024-01-20,extra_value"""
+
+
+@pytest.fixture
+def sample_csv_missing_columns():
+    """Sample CSV missing required columns."""
+    return """experiment_id,mean_normalized_score,Agent
+exp_001,0.854321,Agent A
+exp_002,0.789012,Agent B"""
+
+
+class TestDownloadLeaderboard:
+    """Tests for download_leaderboard function."""
+
+    @patch("app.requests.get")
+    def test_successful_download(self, mock_get, sample_csv_data):
+        """Test successful download and parsing of leaderboard."""
+        # Setup mock GitHub API response
+        mock_responses = create_github_api_response(sample_csv_data)
+        mock_get.side_effect = mock_responses
+
+        # Execute
+        df = download_leaderboard()
+
+        # Assertions
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == 3
+        assert list(df.columns) == DISPLAY_COLUMNS
+        assert mock_get.call_count == 1
+
+    @patch("app.requests.get")
+    def test_data_cleaning_rounding(self, mock_get, sample_csv_data):
+        """Test that numeric columns are properly rounded."""
+        mock_responses = create_github_api_response(sample_csv_data)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Check rounding
+        assert df["mean_normalized_score"].dtype in [float, "float64"]
+        assert df["std_normalized_score"].dtype in [float, "float64"]
+        # Values should be rounded to 3 decimal places
+        assert df.loc[0, "mean_normalized_score"] == 0.912
+        assert df.loc[1, "mean_normalized_score"] == 0.854
+        assert df.loc[2, "mean_normalized_score"] == 0.789
+
+    @patch("app.requests.get")
+    def test_percentage_conversion(self, mock_get, sample_csv_data):
+        """Test that medal percentages are converted from decimal to percentage."""
+        mock_responses = create_github_api_response(sample_csv_data)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Check percentage conversion (0.876543 * 100 = 87.6543, rounded to 87.7)
+        assert df.loc[1, "mean_medal_pct"] == 87.7
+        assert df.loc[0, "mean_medal_pct"] == 92.3
+        assert df.loc[2, "mean_medal_pct"] == 76.5
+
+    @patch("app.requests.get")
+    def test_date_formatting(self, mock_get, sample_csv_data):
+        """Test that dates are properly formatted."""
+        mock_responses = create_github_api_response(sample_csv_data)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Check date formatting
+        assert df.loc[0, "Date"] == "2024-02-01"
+        assert df.loc[1, "Date"] == "2024-01-15"
+        assert df.loc[2, "Date"] == "2024-01-20"
+
+    @patch("app.requests.get")
+    def test_sorting(self, mock_get, sample_csv_data):
+        """Test that dataframe is sorted by mean_normalized_score descending."""
+        mock_responses = create_github_api_response(sample_csv_data)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Check sorting (highest score first)
+        scores = df["mean_normalized_score"].tolist()
+        assert scores == sorted(scores, reverse=True)
+        assert df.loc[0, "experiment_id"] == "exp_003"  # Highest score
+        assert df.loc[2, "experiment_id"] == "exp_002"  # Lowest score
+
+    @patch("app.requests.get")
+    def test_extra_columns_filtered(self, mock_get, sample_csv_with_extra_columns):
+        """Test that extra columns are filtered out."""
+        mock_responses = create_github_api_response(sample_csv_with_extra_columns)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Check that only display columns are present
+        assert list(df.columns) == DISPLAY_COLUMNS
+        assert "extra_col" not in df.columns
+
+    @patch("app.requests.get")
+    def test_missing_columns_error(self, mock_get, sample_csv_missing_columns):
+        """Test that missing required columns raise ValueError."""
+        mock_responses = create_github_api_response(sample_csv_missing_columns)
+        mock_get.side_effect = mock_responses
+
+        with pytest.raises(ValueError, match="Leaderboard is missing expected columns"):
+            download_leaderboard()
+
+    @patch("app.requests.get")
+    def test_http_error(self, mock_get):
+        """Test handling of HTTP errors."""
+        mock_response = Mock()
+        mock_response.raise_for_status.side_effect = requests.HTTPError("404 Not Found")
+        mock_get.return_value = mock_response
+
+        with pytest.raises(requests.HTTPError):
+            download_leaderboard()
+
+    @patch("app.requests.get")
+    def test_network_error(self, mock_get):
+        """Test handling of network errors."""
+        mock_get.side_effect = requests.ConnectionError("Connection failed")
+
+        with pytest.raises(requests.ConnectionError):
+            download_leaderboard()
+
+    @patch("app.requests.get")
+    def test_timeout_handling(self, mock_get):
+        """Test that timeout parameter is passed correctly."""
+        csv_data = "experiment_id,mean_normalized_score,std_normalized_score,mean_medal_pct,sem_medal_pct,Agent,LLM(s) used,Date\nexp_001,0.85,0.01,0.87,0.01,Agent A,GPT-4,2024-01-15"
+        mock_responses = create_github_api_response(csv_data)
+        mock_get.side_effect = mock_responses
+
+        download_leaderboard()
+
+        # Verify timeout was passed
+        assert mock_get.call_count >= 1
+        # Check that timeout is in the first call (API call)
+        call_kwargs = mock_get.call_args_list[0][1]
+        assert call_kwargs["timeout"] == 30
+
+    @patch("app.requests.get")
+    def test_empty_dataframe(self, mock_get):
+        """Test handling of empty CSV (header only)."""
+        csv_data = ",".join(DISPLAY_COLUMNS)  # Header only
+        mock_responses = create_github_api_response(csv_data)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == 0
+        assert list(df.columns) == DISPLAY_COLUMNS
+
+    @patch("app.requests.get")
+    def test_invalid_date_handling(self, mock_get):
+        """Test that invalid dates are handled gracefully."""
+        csv_with_invalid_date = """experiment_id,mean_normalized_score,std_normalized_score,mean_medal_pct,sem_medal_pct,Agent,LLM(s) used,Date
+exp_001,0.854321,0.012345,0.876543,0.009876,Agent A,GPT-4,invalid-date
+exp_002,0.789012,0.023456,0.765432,0.012345,Agent B,Claude-3,2024-01-20"""
+        mock_responses = create_github_api_response(csv_with_invalid_date)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Invalid dates should become NaT and then empty string or NaN
+        assert pd.isna(df.loc[0, "Date"]) or df.loc[0, "Date"] == ""
+        assert df.loc[1, "Date"] == "2024-01-20"
+
+    @patch("app.requests.get")
+    def test_git_lfs_pointer_file(self, mock_get, sample_csv_data):
+        """Test handling of Git LFS pointer files."""
+        # First response: API with LFS pointer
+        mock_responses = create_github_api_response(sample_csv_data, is_lfs_pointer=True)
+        # Add download response for LFS file
+        download_response = Mock()
+        download_response.text = sample_csv_data
+        download_response.raise_for_status = Mock()
+        mock_responses.append(download_response)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        # Should successfully download via download_url
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == 3
+        assert list(df.columns) == DISPLAY_COLUMNS
+        # Should make 2 calls: API call + download_url call
+        assert mock_get.call_count == 2
+
+    @patch("app.requests.get")
+    def test_large_file_download_url(self, mock_get, sample_csv_data):
+        """Test handling of large files that only have download_url."""
+        mock_responses = create_github_api_response(sample_csv_data, use_download_url=True)
+        mock_get.side_effect = mock_responses
+
+        df = download_leaderboard()
+
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == 3
+        assert list(df.columns) == DISPLAY_COLUMNS
+        # Should make 2 calls: API call + download_url call
+        assert mock_get.call_count == 2
+
+
+class TestRefreshLeaderboard:
+    """Tests for refresh_leaderboard function."""
+
+    @patch("app.download_leaderboard")
+    def test_refresh_leaderboard_success(self, mock_download):
+        """Test that refresh_leaderboard returns dataframe and status message."""
+        # Setup mocks
+        mock_df = pd.DataFrame(
+            {
+                "experiment_id": ["exp_001"],
+                "mean_normalized_score": [0.85],
+                "Agent": ["Agent A"],
+            }
+        )
+        mock_download.return_value = mock_df
+
+        # Execute
+        df, status = refresh_leaderboard()
+
+        # Assertions
+        assert df is mock_df
+        assert "Showing data from" in status
+        assert "GitHub" in status
+        # Check that status contains timestamp in expected format (YYYY-MM-DD HH:MM UTC)
+        assert "UTC" in status
+        assert "Last refreshed:" in status
+        # Verify timestamp format (should match pattern YYYY-MM-DD HH:MM)
+        import re
+
+        timestamp_pattern = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2} UTC"
+        assert re.search(timestamp_pattern, status) is not None
+        mock_download.assert_called_once()
+
+    @patch("app.download_leaderboard")
+    def test_refresh_leaderboard_includes_url(self, mock_download):
+        """Test that status message includes the GitHub URL."""
+        mock_df = pd.DataFrame()
+        mock_download.return_value = mock_df
+
+        df, status = refresh_leaderboard()
+
+        assert "github.com" in status.lower() or "GitHub" in status
+        assert "upgini/mle-bench" in status
+
+    @patch("app.download_leaderboard")
+    def test_refresh_leaderboard_propagates_error(self, mock_download):
+        """Test that errors from download_leaderboard are propagated."""
+        mock_download.side_effect = requests.HTTPError("404 Not Found")
+
+        with pytest.raises(requests.HTTPError):
+            refresh_leaderboard()