v0.3.14: Add logging, support alternative trajectory format, fix token columns

- Add logging setup with file handler (logs/app.log) and console output
- Log all parsing errors with full traceback (exc_info=True)
- Add unhandled exception hook to capture crashes
- Add logs/ to .gitignore

- Support alternative trajectory format (Llama 4 Scout style)
- Detect 'trajectory' array format vs 'messages' format
- New _parse_trajectory_format_to_steps() for trajectory format
- Extract api_calls from trajectory length when model_stats missing

- Add ensure_token_columns() to guarantee token columns exist
- Apply ensure_token_columns in load_all_trajectories, load_all_trajectories_calculated
- Fix KeyError: 'completion_tokens' for models with missing token data

.gitignore

@@ -3,6 +3,7 @@
 __pycache__/
 *.pyc
 data/
+logs/
 .DS_Store
 
 

app.py

@@ -1,8 +1,10 @@
 import json
+import logging
 import os
 import random
 import re
 import subprocess
+import sys
 from pathlib import Path
 
 import gradio as gr
@@ -23,6 +25,29 @@ LEADERBOARD_CACHE = DATA_DIR / "swebench_leaderboard_latest.json"
 LITELLM_PRICES_CACHE = DATA_DIR / "litellm_prices.json"
 S3_BUCKET = "s3://swe-bench-experiments/bash-only"
 LITELLM_PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+LOG_DIR = Path("logs")
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+LOG_FILE = LOG_DIR / "app.log"
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.FileHandler(LOG_FILE, encoding="utf-8"),
+        logging.StreamHandler(sys.stdout),
+    ],
+    force=True,
+)
+
+
+def _log_unhandled(exc_type, exc_value, exc_traceback):
+    if issubclass(exc_type, KeyboardInterrupt):
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+        return
+    logging.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
+sys.excepthook = _log_unhandled
 
 _litellm_prices_cache = None
 _trajectories_cache = {}
@@ -106,6 +131,11 @@ def parse_trajectory_to_steps(traj_path: Path, model_name: str) -> list[dict]:
         data = json.load(f)
 
     messages = data.get("messages", [])
+    trajectory_data = data.get("trajectory", [])
+
+    if not messages and trajectory_data:
+        return _parse_trajectory_format_to_steps(trajectory_data, model_name)
+
     if not messages:
         return []
 
@@ -151,6 +181,40 @@ def parse_trajectory_to_steps(traj_path: Path, model_name: str) -> list[dict]:
     return steps
 
 
+def _parse_trajectory_format_to_steps(trajectory_data: list, model_name: str) -> list[dict]:
+    """
+    Parse alternative trajectory format (with "trajectory" array) into steps.
+    """
+    count_tokens, _ = get_tokenizer(model_name)
+
+    steps = []
+    for i, traj_step in enumerate(trajectory_data):
+        query = traj_step.get("query", [])
+        response_text = traj_step.get("response", "")
+        observation_text = traj_step.get("observation", "")
+
+        system_user_tokens = 0
+        if i == 0:
+            for q in query:
+                content = q.get("content", "")
+                if isinstance(content, list):
+                    content = json.dumps(content)
+                system_user_tokens += count_tokens(str(content))
+
+        completion_tokens = count_tokens(str(response_text)) if response_text else 0
+        observation_tokens = count_tokens(str(observation_text)) if observation_text else None
+
+        step = {
+            "model": model_name,
+            "system_user": system_user_tokens,
+            "completion": completion_tokens,
+            "observation": observation_tokens,
+        }
+        steps.append(step)
+
+    return steps
+
+
 def get_default_overhead(model_name: str) -> float:
     """Get default tokenizer overhead for model provider"""
     model_lower = model_name.lower() if model_name else ""
@@ -212,13 +276,33 @@ def apply_no_cache(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def ensure_token_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Ensure token-related columns exist and are numeric."""
+    if df is None or df.empty:
+        return df
+    df = df.copy()
+    required = [
+        "prompt_tokens",
+        "completion_tokens",
+        "cache_read_tokens",
+        "cache_creation_tokens",
+    ]
+    for col in required:
+        if col not in df.columns:
+            df[col] = 0
+        df[col] = pd.to_numeric(df[col], errors="coerce").fillna(0).astype(int)
+    if "total_tokens" in df.columns:
+        df["total_tokens"] = pd.to_numeric(df["total_tokens"], errors="coerce").fillna(0).astype(int)
+    return df
+
+
 def load_all_trajectories_calculated(folder: str) -> pd.DataFrame:
     """Load trajectories with self-calculated token counts using calculate_routing_tokens"""
     global _calculated_tokens_cache
 
     cache_key = f"calculated_{folder}"
     if cache_key in _calculated_tokens_cache:
-        return _calculated_tokens_cache[cache_key]
+        return ensure_token_columns(_calculated_tokens_cache[cache_key])
 
     trajectory_steps = load_all_trajectory_steps(folder)
 
@@ -251,9 +335,9 @@ def load_all_trajectories_calculated(folder: str) -> pd.DataFrame:
                 "cache_creation_tokens": cache_creation,
             })
         except Exception as e:
-            print(f"Error calculating tokens for {instance_id}: {e}")
+            logging.error("Error calculating tokens for %s: %s", instance_id, e, exc_info=True)
 
-    df = pd.DataFrame(rows)
+    df = ensure_token_columns(pd.DataFrame(rows))
     _calculated_tokens_cache[cache_key] = df
     return df
 
@@ -301,7 +385,7 @@ def load_all_trajectory_steps(folder: str) -> dict[str, list[dict]]:
             if steps:
                 result[instance_id] = steps
         except Exception as e:
-            print(f"Error parsing steps for {traj_path}: {e}")
+            logging.error("Error parsing steps for %s: %s", traj_path, e, exc_info=True)
 
     _trajectory_steps_cache[cache_key] = result
     return result
@@ -519,10 +603,29 @@ def parse_trajectory(traj_path: Path) -> dict:
     model_config = config.get("model", {})
     model_name = model_config.get("cost_calc_model_override", model_config.get("model_name", ""))
 
+    trajectory_steps = data.get("trajectory", [])
+    is_trajectory_format = len(trajectory_steps) > 0 and "messages" not in data
+
+    if is_trajectory_format and not model_name:
+        for step in trajectory_steps:
+            query = step.get("query", [])
+            for q in query:
+                if q.get("role") == "system":
+                    content = q.get("content", "")
+                    if "llama" in content.lower() or "meta" in content.lower():
+                        model_name = "llama"
+                        break
+            if model_name:
+                break
+
+    api_calls = model_stats.get("api_calls", 0)
+    if api_calls == 0 and is_trajectory_format:
+        api_calls = len(trajectory_steps)
+
     result = {
         "instance_id": data.get("instance_id", traj_path.stem),
         "model_name": model_name,
-        "api_calls": model_stats.get("api_calls", 0),
+        "api_calls": api_calls,
         "instance_cost": model_stats.get("instance_cost", 0),
         "prompt_tokens": 0,
         "completion_tokens": 0,
@@ -555,7 +658,7 @@ def load_all_trajectories(folder: str) -> pd.DataFrame:
     global _trajectories_cache
 
     if folder in _trajectories_cache:
-        return _trajectories_cache[folder]
+        return ensure_token_columns(_trajectories_cache[folder])
 
     output_dir = TRAJS_DIR / folder
 
@@ -574,9 +677,9 @@ def load_all_trajectories(folder: str) -> pd.DataFrame:
         try:
             rows.append(parse_trajectory(traj_path))
         except Exception as e:
-            print(f"Error parsing {traj_path}: {e}")
+            logging.error("Error parsing %s: %s", traj_path, e, exc_info=True)
 
-    df = pd.DataFrame(rows)
+    df = ensure_token_columns(pd.DataFrame(rows))
     _trajectories_cache[folder] = df
     return df
 
@@ -981,8 +1084,8 @@ def get_prices_for_folder(folder: str) -> tuple[dict, str]:
     return result, model_hint
 
 
-def on_row_select(evt: gr.SelectData, df: pd.DataFrame):
-    if evt.index is None:
+def _build_selection_payload(row_idx: int | None, df: pd.DataFrame):
+    if df is None or df.empty or row_idx is None:
         return (
             "", "",
             gr.update(visible=False),
@@ -994,7 +1097,6 @@ def on_row_select(evt: gr.SelectData, df: pd.DataFrame):
             gr.update(value=1.0),
         )
 
-    row_idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index
     row = df.iloc[row_idx]
     folder = row["folder"]
     name = row["name"]
@@ -1023,6 +1125,18 @@ def on_row_select(evt: gr.SelectData, df: pd.DataFrame):
     )
 
 
+def on_row_select(evt: gr.SelectData, df: pd.DataFrame):
+    row_idx = None
+    if evt is not None and evt.index is not None:
+        row_idx = evt.index[0] if isinstance(evt.index, (list, tuple)) else evt.index
+    return _build_selection_payload(row_idx, df)
+
+
+def select_first_row(df: pd.DataFrame):
+    default_idx = 0 if df is not None and not df.empty else None
+    return _build_selection_payload(default_idx, df)
+
+
 def create_routed_token_chart(base_tokens: dict, additional_models: list):
     """
     Create grouped bar chart for tokens by type, comparing base vs additional models.
@@ -1155,8 +1269,8 @@ def build_app():
     with gr.Blocks(title="SWE-bench Routing Cost Calculator") as app:
         trajectories_state = gr.State(None)
 
-        gr.Markdown("# 🧮 SWE-bench Bash-Only Leaderboard `v0.3.9`")
-        gr.Markdown("Select a model to use as base for cost analysis")
+        gr.Markdown("# 🧮 SWE-bench Bash-Only Leaderboard `v0.3.14`")
+        gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
 
         with gr.Row():
             with gr.Column(scale=3):
@@ -1165,6 +1279,7 @@ def build_app():
                     label="Bash-Only Leaderboard",
                     interactive=False,
                     wrap=True,
+                    elem_id="leaderboard-table",
                 )
 
                 with gr.Column(visible=False) as analysis_section:
@@ -1777,7 +1892,23 @@ def build_app():
             outputs=[selected_folder, selected_name, analyze_btn, price_input, price_cache_read, price_cache_creation, price_completion, detected_model, thinking_overhead],
         )
 
+        app.load(
+            fn=select_first_row,
+            inputs=[leaderboard_table],
+            outputs=[selected_folder, selected_name, analyze_btn, price_input, price_cache_read, price_cache_creation, price_completion, detected_model, thinking_overhead],
+            js="""
+(data) => {
+    const row = gradioApp()?.querySelector('#leaderboard-table table tbody tr');
+    if (row) {
+        row.click();
+    }
+    return data;
+}
+""",
+        )
+
         def load_and_analyze(folder, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache, progress=gr.Progress()):
+            progress(0, desc="Ready")
             empty_result = (
                 "",
                 gr.update(visible=False),
@@ -1789,10 +1920,12 @@ def build_app():
             )
 
             if not folder:
+                progress(1, desc="No folder selected")
                 yield empty_result
                 return
 
             if not check_trajectories_downloaded(folder):
+                progress(0.1, desc="Preparing download")
                 yield (
                     "⏳ Downloading trajectories...",
                     gr.update(visible=False),
@@ -1802,8 +1935,10 @@ def build_app():
                     None,
                     gr.update(visible=False),
                 )
+                progress(0.3, desc="Downloading")
                 status, _ = download_trajectories_from_s3(folder)
                 if "❌" in status:
+                    progress(1, desc="Download failed")
                     yield (
                         status,
                         gr.update(visible=False),
@@ -1814,6 +1949,7 @@ def build_app():
                         gr.update(visible=False),
                     )
                     return
+            progress(0.45, desc="Loading trajectories")
 
             yield (
                 "⏳ Loading trajectories...",
@@ -1825,15 +1961,19 @@ def build_app():
                 gr.update(visible=False),
             )
 
-            df_meta = load_all_trajectories(folder)
-            df_calc = load_all_trajectories_calculated(folder)
+            progress(0.6, desc="Reading metadata")
+            df_meta = ensure_token_columns(load_all_trajectories(folder))
+            progress(0.7, desc="Reading calculated")
+            df_calc = ensure_token_columns(load_all_trajectories_calculated(folder))
             df_calc["api_calls"] = df_meta["api_calls"].values
             df_calc["instance_cost"] = df_meta["instance_cost"].values
+            progress(0.8, desc="Reading steps")
             trajectory_steps = load_all_trajectory_steps(folder)
 
             state_data = {"meta": df_meta, "calculated": df_calc, "folder": folder, "steps": trajectory_steps}
 
             if df_meta.empty:
+                progress(1, desc="No trajectories found")
                 yield (
                     "❌ No trajectories found",
                     gr.update(visible=False),
@@ -1845,6 +1985,7 @@ def build_app():
                 )
                 return
 
+            progress(0.9, desc="Building charts")
             fig_steps, fig_cost, _, _, _ = create_basic_histograms(
                 df_meta, input_price, cache_read_price, cache_creation_price, completion_price
             )
@@ -1867,6 +2008,7 @@ def build_app():
                 df_calc_processed, input_price, cache_read_price, cache_creation_price, completion_price
             )
 
+            progress(1, desc="Done")
             yield (
                 f"✅ Loaded {len(df_meta)} trajectories",
                 gr.update(visible=True),