Upload from GitHub Actions: Merge pull request #28 from datenlabor-bmz/jn-dev

.github/workflows/nightly-evals.yml

@@ -21,14 +21,17 @@ jobs:
       - name: Install dependencies
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
-          uv sync --frozen --extra dev
+          # Use the `dev` dependency group defined in pyproject.toml
+          uv sync --frozen --group dev
           
       - name: Run evaluations
         env:
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
-          N_SENTENCES: 20
-          MAX_LANGUAGES: 150
+          N_SENTENCES: 10
+          # Keep these aligned with defaults in evals/main.py for comparability
+          N_LANGUAGES: 1000
+          N_MODELS: 40
         run: |
           uv run huggingface-cli login --token ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
           uv run evals/download_data.py

README.md

@@ -47,7 +47,8 @@ _AI model evaluations for every language in the world_
 
 ### Local Development
 ```bash
-uv run --extra dev evals/main.py
+uv sync --group dev
+uv run evals/main.py
 ```
 
 ## Explore

evals/main.py

@@ -39,7 +39,13 @@ async def evaluate():
     # Load cached results and filter out completed combinations
     old_results = load("results-detailed")
     if not old_results.empty:
-        completed = set(old_results[["task", "model", "bcp_47", "sentence_nr"]].apply(tuple, axis=1))
+        # Only treat status==\"ok\" (or missing status) as completed.
+        if "status" in old_results.columns:
+            ok_mask = old_results["status"].isna() | (old_results["status"] == "ok")
+            completed_df = old_results.loc[ok_mask, ["task", "model", "bcp_47", "sentence_nr"]]
+        else:
+            completed_df = old_results[["task", "model", "bcp_47", "sentence_nr"]]
+        completed = set(completed_df.apply(tuple, axis=1))
         combis = combis[~combis.apply(lambda row: tuple(row) in completed, axis=1)]
 
     print(f"Running {len(combis)} evaluation tasks...")
@@ -57,16 +63,24 @@ async def evaluate():
     results = [r for batch in batch_results for result in batch for r in result]
     results = pd.DataFrame(results) if results else pd.DataFrame(columns=["task", "model", "bcp_47", "metric", "sentence_nr", "score", "origin"])
     
-    # Merge with cached results (immutable log)
+    # Merge with cached results (immutable log, prefer latest results on conflict)
     all_results = pd.concat([old_results, results]).drop_duplicates(
-        subset=["task", "model", "bcp_47", "metric", "sentence_nr"]
+        subset=["task", "model", "bcp_47", "metric", "sentence_nr"],
+        keep="last",
     ) if not old_results.empty else results
     
-    # Filter to current models × languages and aggregate
+    # Filter to current models × languages and aggregate.
+    # Only aggregate over successful evaluations (status == \"ok\" or missing).
     current_models = set(models.iloc[:n_models]["id"])
     current_languages = set(languages.head(n_languages)["bcp_47"])
+    if "status" in all_results.columns:
+        valid_mask = all_results["status"].isna() | (all_results["status"] == "ok")
+        valid_results = all_results[valid_mask]
+    else:
+        valid_results = all_results
+
     results_agg = (
-        all_results[all_results["model"].isin(current_models) & all_results["bcp_47"].isin(current_languages)]
+        valid_results[valid_results["model"].isin(current_models) & valid_results["bcp_47"].isin(current_languages)]
         .groupby(["model", "bcp_47", "task", "metric"])
         .agg({"score": "mean", "origin": "first"})
         .reset_index()

evals/models.py

@@ -44,26 +44,17 @@ important_models = [
     "google/gemini-2.5-flash-lite",  # 0.3$
     "google/gemma-3-27b-it",  # 0.2$
     # "x-ai/grok-4", # $15
-    "x-ai/grok-4.1-fast:free", #free for now
     "x-ai/grok-4-fast", 
-    # "x-ai/grok-3", # $15
     "cohere/command-a",
-    "qwen/qwen3-32b",
-    "qwen/qwen3-235b-a22b",
+    # "qwen/qwen3-32b",
+    # "qwen/qwen3-235b-a22b",
     "qwen/qwen3-30b-a3b",  # 0.29$
-    # "qwen/qwen-turbo", # 0.2$; recognizes "inappropriate content"
-    # "qwen/qwq-32b",  # 0.2$
-    # "qwen/qwen-2.5-72b-instruct",  # 0.39$
-    # "qwen/qwen-2-72b-instruct",  # 0.9$
     "deepseek/deepseek-v3.2-exp",
     "microsoft/phi-4",  # 0.07$
     "amazon/nova-premier-v1", # 12.5$
     "amazon/nova-pro-v1",  # 0.09$
     "moonshotai/kimi-k2",  # 0.6$
-    # "moonshotai/kimi-k2-thinking", # 2.5$
     "baidu/ernie-4.5-300b-a47b",
-    # "baidu/ernie-4.5-21b-a3b-thinking",
-    "z-ai/glm-4.6", # 1.75$
 ]
 
 blocklist = [

evals/tasks.py

@@ -117,6 +117,7 @@ async def translate_and_evaluate(model, bcp_47, sentence_nr, mode="from"):
             )
     else:
         prediction = await query(model, translation_prompt)
+    status = "ok" if prediction else "error"
     if prediction:
         bleu_score = bleu.compute(
             predictions=[prediction],
@@ -141,6 +142,7 @@ async def translate_and_evaluate(model, bcp_47, sentence_nr, mode="from"):
             "sentence_nr": sentence_nr,
             "prompt": translation_prompt,
             "response": prediction,
+            "status": status,
         }
         for metric, score in (
             ("bleu", bleu_score["bleu"]),
@@ -171,6 +173,7 @@ Text:
 """
     response = await query(model, prompt)
     pred = response.lower().strip() if response else ""
+    status = "ok" if pred else "error"
     true = test_paragraph.topic.lower().strip()
     others = [t for t in top_topics if t != true]
     acc = (
@@ -193,6 +196,7 @@ Text:
             "sentence_nr": nr,
             "prompt": prompt,
             "response": pred,
+            "status": status,
         }
     ]
 
@@ -256,6 +260,7 @@ async def mmlu_and_evaluate(model, language_bcp_47, nr):
     response = await query(model, prompt)
     final_response = extract_mc_response(response)
     acc = int(final_response == task["answer"]) if final_response else 0
+    status = "ok" if final_response else "error"
 
     return [
         {
@@ -268,6 +273,7 @@ async def mmlu_and_evaluate(model, language_bcp_47, nr):
             "sentence_nr": nr,
             "prompt": prompt,
             "response": response,
+            "status": status,
         }
     ]
 
@@ -280,6 +286,7 @@ async def arc_and_evaluate(model, language_bcp_47, nr):
     response = await query(model, prompt)
     final_response = extract_mc_response(response)
     acc = int(final_response == task["answer"]) if final_response else 0
+    status = "ok" if final_response else "error"
     return [
         {
             "model": model,
@@ -291,6 +298,7 @@ async def arc_and_evaluate(model, language_bcp_47, nr):
             "sentence_nr": nr,
             "prompt": prompt,
             "response": response,
+            "status": status,
         }
     ]
 
@@ -323,6 +331,7 @@ async def truthfulqa_and_evaluate(model, language_bcp_47, nr):
     response = await query(model, prompt)
     final_response = extract_mc_response(response)
     acc = int(final_response.upper() == answer) if final_response else 0
+    status = "ok" if final_response else "error"
     return [
         {
             "model": model,
@@ -334,6 +343,7 @@ async def truthfulqa_and_evaluate(model, language_bcp_47, nr):
             "sentence_nr": nr,
             "prompt": prompt,
             "response": response,
+            "status": status,
         }
     ]
 
@@ -358,6 +368,7 @@ async def mgsm_and_evaluate(model, language_bcp_47, nr):
         if number
         else 0
     )
+    status = "ok" if number else "error"
     return [
         {
             "model": model,
@@ -369,6 +380,7 @@ async def mgsm_and_evaluate(model, language_bcp_47, nr):
             "sentence_nr": nr,
             "prompt": prompt,
             "response": response,
+            "status": status,
         }
     ]
 

pyproject.toml

@@ -37,4 +37,4 @@ dev = [
     "tiktoken>=0.9.0",
     "tqdm>=4.67.1",
     "transformers>=4.51.3",
-]
+]