Upload folder using huggingface_hub

afridialeval/import_data.py

@@ -37,8 +37,6 @@ def parse_filename(filename: str) -> Dict[str, str]:
 
     known_models = {
         "gpt_5_1": "gpt-5.1",
-        "qwen_3_5_27b": "qwen-3.5-27b",
-        "qwen_3_5_122b": "qwen-3.5-122b",
         "gemma_3_27b_it": "gemma-3-27b-it",
         "gemini-3-flash-preview": "gemini-3-flash-preview",
     }

afridialeval/models.py

@@ -3,7 +3,6 @@
 Codename mapping (blind, consistent):
   Kifaru (🦏) → gpt-5.1
   Nyati  (🦬) → gemini-3-flash-preview
-  Tembo  (🐘) → qwen-3.5-27b
   Chui   (🐆) → gemma-3-27b-it
 """
 
@@ -21,14 +20,12 @@ CODENAMES = {
     "chui": "gemma-3-27b-it",
     "kifaru": "gpt-5.1",
     "nyati": "gemini-3-flash-preview",
-    "tembo": "qwen-3.5-27b",
 }
 MODEL_TO_CODENAME = {v: k for k, v in CODENAMES.items()}
 CODENAME_DISPLAY = {
     "chui": "🐆 Chui",
     "kifaru": "🦏 Kifaru",
     "nyati": "🦬 Nyati",
-    "tembo": "🐘 Tembo",
 }
 
 

afridialeval/templates/annotate.html

@@ -64,7 +64,7 @@
             <!-- Step 1: Read all variants -->
             <div class="step" id="step-1">
                 <h2>Step 1: Read all versions</h2>
-                <p class="instruction">Each version (🐆 Chui, 🦏 Kifaru, 🦬 Nyati, 🐘 Tembo) was generated by a different system. Read them carefully.</p>
+                <p class="instruction">Each version (🐆 Chui, 🦏 Kifaru, 🦬 Nyati) was generated by a different system. Read them carefully.</p>
 
                 <div class="dialogue-trio">
                     {% for v in variants %}

pyproject.toml

@@ -6,6 +6,8 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "azure-identity>=1.25.3",
+    "huggingface-hub>=0.25.0",
     "openai>=2.31.0",
+    "python-dotenv>=1.0.0",
     "tqdm>=4.67.3",
 ]

run_pipeline.sh

@@ -48,15 +48,6 @@ case "$MODE" in
     # Yoruba
     run_one "Yoruba" "Nigeria - Lagos" "$@"
     run_one "Yoruba" "Benin - Porto-Novo" "$@"
-    # Somali
-    run_one "Somali" "Somalia - Mogadishu" "$@"
-    run_one "Somali" "Kenya - Nairobi" "$@"
-    # Tsonga
-    run_one "Tsonga" "South Africa - Johannesburg" "$@"
-    run_one "Tsonga" "Mozambique - Maputo" "$@"
-    # Amharic
-    run_one "Amharic" "Ethiopia - Addis Ababa" "$@"
-    run_one "Amharic" "Eritrea - Asmara" "$@"
     ;;
   one)
     LANGUAGE="$1"

src/generator.py

@@ -1,18 +1,23 @@
 import hashlib
 import json
+import os
 import re
 import time
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
 from azure.identity import (
     AzureCliCredential,
     ChainedTokenCredential,
     ManagedIdentityCredential,
     get_bearer_token_provider,
 )
-from openai import AzureOpenAI
+from openai import AzureOpenAI, OpenAI
 from tqdm import tqdm
 
 from src.config import (
@@ -72,7 +77,19 @@ class Generator:
         self.client = self._build_client()
         self.prompt_cache: Dict[str, str] = self._load_prompt_cache()
 
-    def _build_client(self) -> AzureOpenAI:
+    def _build_client(self) -> Union[AzureOpenAI, OpenAI]:
+        if self.model_config.backend == "huggingface":
+            hf_token = os.environ.get("HF_TOKEN")
+            if not hf_token:
+                raise RuntimeError(
+                    "HF_TOKEN environment variable is required for Hugging Face models. "
+                    "Get a token at https://huggingface.co/settings/tokens"
+                )
+            return OpenAI(
+                base_url="https://router.huggingface.co/hf-inference/v1",
+                api_key=hf_token,
+            )
+
         token_provider = get_bearer_token_provider(
             ChainedTokenCredential(
                 AzureCliCredential(),
@@ -150,10 +167,6 @@ class Generator:
         if response_format is not None and self.model_config.is_openai_compatible:
             kwargs["response_format"] = response_format
 
-        # Disable thinking for Qwen 3.5 models to avoid slow reasoning tokens
-        if "qwen" in self.model_alias.lower() and "3.5" in self.model_alias:
-            kwargs["extra_body"] = {"chat_template_kwargs": {"enable_thinking": False}}
-
         response = self.client.chat.completions.create(**kwargs)
         content = response.choices[0].message.content or ""
 

src/model_registry.py

@@ -9,6 +9,7 @@ class ModelConfig:
     api_version: str
     is_openai_compatible: bool = True
     endpoint_override: str = ""  # If set, use this TRAPI endpoint instead of default
+    backend: str = "trapi"  # "trapi" or "huggingface"
 
 
 MODELS: Dict[str, ModelConfig] = {
@@ -19,81 +20,18 @@ MODELS: Dict[str, ModelConfig] = {
         is_openai_compatible=True,
         endpoint_override="https://trapi.research.microsoft.com/msrc/shared",
     ),
-    "gpt-5.4": ModelConfig(
-        alias="gpt-5.4",
-        deployment_name="gpt-5.4_2026-03-05",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=True,
-    ),
     "gpt-5.4-mini": ModelConfig(
         alias="gpt-5.4-mini",
         deployment_name="gpt-5.4-mini_2026-03-17",
         api_version="2025-04-01-preview",
         is_openai_compatible=True,
     ),
-    "gpt-4o": ModelConfig(
-        alias="gpt-4o",
-        deployment_name="gpt-4o_2024-11-20",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=True,
-    ),
-    "llama-3.3-70B": ModelConfig(
-        alias="llama-3.3-70B",
-        deployment_name="Llama-3.3-70B-Instruct_5",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=False,
-    ),
     "gemma-3-27b-it": ModelConfig(
         alias="gemma-3-27b-it",
-        deployment_name="unsloth/gemma-3-27b-it",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=False,
-    ),
-    "gemma-3-4b-it": ModelConfig(
-        alias="gemma-3-4b-it",
-        deployment_name="unsloth/gemma-3-4b-it",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=False,
-    ),
-    "qwen-3.5-122b": ModelConfig(
-    alias="qwen-3.5-122b",
-    deployment_name="Qwen/Qwen3.5-122B-A10B",
-    api_version="2025-04-01-preview",
-    is_openai_compatible=False,
-    endpoint_override="https://trapi.research.microsoft.com/msrc/shared",
-    ),
-    "qwen-3.5-27b": ModelConfig(
-        alias="qwen-3.5-27b",
-        deployment_name="Qwen/Qwen3.5-27B",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=False,
-        endpoint_override="https://trapi.research.microsoft.com/msrc/shared",
-    ),
-    "qwen-3.5-397b": ModelConfig(
-        alias="qwen-3.5-397b",
-        deployment_name="Qwen/Qwen3.5-397B-A17B-GPTQ-Int4",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=False,
-        endpoint_override="https://trapi.research.microsoft.com/msrc/shared",
-    ),
-    "qwen-3.5-9b": ModelConfig(
-        alias="qwen-3.5-9b",
-        deployment_name="Qwen/Qwen3.5-9B",
-        api_version="2025-04-01-preview",
+        deployment_name="google/gemma-3-27b-it",
+        api_version="",
         is_openai_compatible=False,
-        endpoint_override="https://trapi.research.microsoft.com/msrc/shared",
-    ),
-    "Phi-4-reasoning": ModelConfig(
-        alias="Phi-4-reasoning",
-        deployment_name="gcr-phi-4-reasoning",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=True,
-    ),
-    "Phi-4-reasoning-mini": ModelConfig(
-        alias="Phi-4-reasoning-mini",
-        deployment_name="gcr-phi-4-mini-reasoning",
-        api_version="2025-04-01-preview",
-        is_openai_compatible=True,
+        backend="huggingface",
     ),
     "gemini-3-flash-preview": ModelConfig(
         alias="gemini-3-flash-preview",
@@ -104,7 +42,7 @@ MODELS: Dict[str, ModelConfig] = {
 }
 
 
-DEFAULT_MODEL_ALIAS = "gpt-4o"
+DEFAULT_MODEL_ALIAS = "gpt-5.4-mini"
 
 
 def get_model_config(alias: str) -> ModelConfig:

src/orchestrator.py

@@ -20,7 +20,6 @@ DEFAULT_COMPARE_MODELS = [
     "gpt-5.1",
     "gemma-3-27b-it",
     "gemini-3-flash-preview",
-    "qwen-3.5-27b",
 ]