Initial Commit

.gitignore

@@ -0,0 +1,26 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+venv/
+.venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Secrets (never commit these!)
+.env
+*.env
+
+# Leaderboard data (regenerated on Space)
+benchmark_data/leaderboard.json

README.md

@@ -0,0 +1,58 @@
+# Rabbinic Hebrew/Aramaic Embedding Evaluation
+
+A Hugging Face Space for evaluating embedding models on Rabbinic Hebrew and Aramaic texts using cross-lingual retrieval benchmarks.
+
+## Overview
+
+This tool helps identify which embedding models best capture the semantics of Rabbinic Hebrew and Aramaic by measuring how well they align source texts with their English translations. Models that excel at this task are likely to produce high-quality embeddings for untranslated texts.
+
+## Evaluation Approach
+
+Given a Hebrew/Aramaic text, the benchmark tests whether the embedding model can find its correct English translation from a pool of candidates. This cross-lingual retrieval task measures semantic alignment across languages.
+
+### Metrics
+
+| Metric | Description |
+|--------|-------------|
+| **Recall@1** | % of queries where correct translation is the top result |
+| **Recall@5** | % where correct translation is in top 5 results |
+| **Recall@10** | % where correct translation is in top 10 results |
+| **MRR** | Mean Reciprocal Rank (average of 1/rank of correct answer) |
+
+## Corpus
+
+The benchmark includes diverse texts from Sefaria with English translations:
+
+- **Talmud**: Bavli and Yerushalmi (Aramaic + Hebrew)
+- **Mishnah**: All tractates (Rabbinic Hebrew)
+- **Midrash**: Midrash Rabbah (Hebrew/Aramaic)
+- **Tanakh Commentary**: Rashi and Ramban on Tanakh (Hebrew)
+- **Hasidic/Kabbalistic**: Likutei Moharan, Tomer Devorah (Hebrew)
+- **Halacha**: Sefer HaHinuch, Intro to Shev Shmateta (Hebrew)
+
+## Usage
+
+1. Select a model from the curated list or enter any Hugging Face model ID
+2. Click "Run Evaluation"
+3. View results and compare with the leaderboard
+
+## Models
+
+### Curated Models
+- `intfloat/multilingual-e5-large`
+- `sentence-transformers/paraphrase-multilingual-mpnet-base-v2`
+- `BAAI/bge-m3`
+
+You can also evaluate any sentence-transformer compatible model from Hugging Face Hub.
+
+## Local Development
+
+```bash
+pip install -r requirements.txt
+python app.py
+```
+
+## License
+
+MIT
+

app.py

@@ -0,0 +1,498 @@
+"""
+Gradio interface for Rabbinic Hebrew/Aramaic Embedding Evaluation.
+
+A Hugging Face Space for evaluating embedding models on cross-lingual
+retrieval between Hebrew/Aramaic source texts and English translations.
+"""
+
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+
+import gradio as gr
+import pandas as pd
+import plotly.graph_objects as go
+
+from data_loader import load_benchmark_dataset, get_benchmark_stats
+from models import (
+    CURATED_MODELS,
+    API_MODELS,
+    ALL_MODELS,
+    get_curated_model_choices,
+    get_api_model_choices,
+    get_all_model_choices,
+    load_model,
+    validate_model_id,
+    is_api_model,
+    requires_api_key,
+    api_key_optional,
+    get_api_key_type,
+    get_api_key_env_var,
+)
+from evaluation import (
+    EvaluationResults,
+    evaluate_model,
+    evaluate_model_streaming,
+    compute_similarity_matrix,
+    get_rank_distribution,
+)
+
+# Paths
+BENCHMARK_PATH = "benchmark_data/benchmark.json"
+LEADERBOARD_PATH = "benchmark_data/leaderboard.json"
+
+# Global state
+_benchmark_data = None
+_leaderboard = []
+
+
+def load_benchmark():
+    """Load benchmark data, with fallback to sample data."""
+    global _benchmark_data
+    
+    if _benchmark_data is not None:
+        return _benchmark_data
+    
+    try:
+        _benchmark_data = load_benchmark_dataset(BENCHMARK_PATH)
+        print(f"Loaded {len(_benchmark_data)} benchmark pairs")
+    except FileNotFoundError:
+        print("Benchmark not found, using sample data")
+        # Create minimal sample data for testing
+        _benchmark_data = [
+            {
+                "ref": "Sample.1",
+                "he": "בראשית ברא אלהים את השמים ואת הארץ",
+                "en": "In the beginning God created the heaven and the earth",
+                "category": "Sample",
+            },
+            {
+                "ref": "Sample.2",
+                "he": "והארץ היתה תהו ובהו וחשך על פני תהום",
+                "en": "And the earth was without form, and void; and darkness was upon the face of the deep",
+                "category": "Sample",
+            },
+        ]
+    
+    return _benchmark_data
+
+
+def load_leaderboard():
+    """Load saved leaderboard results."""
+    global _leaderboard
+    
+    try:
+        with open(LEADERBOARD_PATH, "r") as f:
+            _leaderboard = json.load(f)
+    except FileNotFoundError:
+        _leaderboard = []
+    
+    return _leaderboard
+
+
+def save_leaderboard():
+    """Save leaderboard to file."""
+    global _leaderboard
+    
+    Path(LEADERBOARD_PATH).parent.mkdir(parents=True, exist_ok=True)
+    with open(LEADERBOARD_PATH, "w") as f:
+        json.dump(_leaderboard, f, indent=2)
+
+
+def add_to_leaderboard(results: EvaluationResults):
+    """Add evaluation results to leaderboard."""
+    global _leaderboard
+    
+    entry = results.to_dict()
+    entry["timestamp"] = datetime.now().isoformat()
+    
+    # Remove existing entry for same model
+    _leaderboard = [e for e in _leaderboard if e["model_id"] != results.model_id]
+    _leaderboard.append(entry)
+    
+    # Sort by MRR descending
+    _leaderboard.sort(key=lambda x: x["mrr"], reverse=True)
+    
+    save_leaderboard()
+
+
+def format_leaderboard_df():
+    """Format leaderboard as pandas DataFrame for display."""
+    load_leaderboard()
+    
+    if not _leaderboard:
+        return pd.DataFrame(columns=[
+            "#", "Model", "MRR", "R@1", "R@5", "R@10", 
+            "Bitext", "TrueSim", "RandSim", "N"
+        ])
+    
+    rows = []
+    for i, entry in enumerate(_leaderboard, 1):
+        rows.append({
+            "#": i,
+            "Model": entry.get("model_name", entry["model_id"]),
+            "MRR": f"{entry['mrr']:.3f}",
+            "R@1": f"{entry['recall_at_1']:.1%}",
+            "R@5": f"{entry['recall_at_5']:.1%}",
+            "R@10": f"{entry['recall_at_10']:.1%}",
+            "Bitext": f"{entry['bitext_accuracy']:.1%}",
+            "TrueSim": f"{entry['avg_true_pair_similarity']:.3f}",
+            "RandSim": f"{entry['avg_random_pair_similarity']:.3f}",
+            "N": entry["num_pairs"],
+        })
+    
+    return pd.DataFrame(rows)
+
+
+def run_evaluation(
+    model_choice: str,
+    custom_model_id: str,
+    api_key: str,
+    max_pairs: int,
+):
+    """
+    Run evaluation for the selected model (generator for streaming status updates).
+    
+    Args:
+        model_choice: Selected curated model or "custom"
+        custom_model_id: Custom model ID if selected
+        api_key: API key for API-based models
+        max_pairs: Maximum pairs to evaluate
+        
+    Yields:
+        Tuples of (status, results, leaderboard)
+    """
+    # Helper to yield status updates
+    def status_update(msg):
+        return (msg, gr.update(), gr.update())
+    
+    # Determine which model to use
+    if model_choice == "custom":
+        model_id = custom_model_id.strip()
+        is_valid, error = validate_model_id(model_id)
+        if not is_valid:
+            yield (
+                f"❌ {error}",
+                f"❌ Invalid model ID: {error}",
+                format_leaderboard_df(),
+            )
+            return
+    else:
+        model_id = model_choice
+    
+    # Check if API key is required but not provided
+    if requires_api_key(model_id):
+        api_key = api_key.strip() if api_key else ""
+        env_var = get_api_key_env_var(model_id)
+        key_type = get_api_key_type(model_id)
+        
+        # Skip API key check for models that support Application Default Credentials
+        if not api_key and not os.environ.get(env_var) and not api_key_optional(model_id):
+            yield (
+                "❌ API key required",
+                f"❌ API key required for {model_id}. Please enter your {key_type.upper()} API key or set the {env_var} environment variable.",
+                format_leaderboard_df(),
+            )
+            return
+    
+    yield status_update(f"⏳ Loading benchmark data...")
+    benchmark = load_benchmark()
+    
+    if max_pairs and max_pairs < len(benchmark):
+        benchmark = benchmark[:max_pairs]
+    
+    yield status_update(f"⏳ Loading model: {model_id}...")
+    
+    try:
+        # Pass API key for API-based models
+        model = load_model(model_id, api_key=api_key if api_key else None)
+    except Exception as e:
+        yield (
+            "❌ Model load failed",
+            f"❌ Failed to load model: {str(e)}",
+            format_leaderboard_df(),
+        )
+        return
+    
+    # Stream progress updates during evaluation
+    try:
+        results = None
+        for item in evaluate_model_streaming(model, benchmark, batch_size=32):
+            if isinstance(item, str):
+                # Progress update
+                yield status_update(item)
+            else:
+                # Final results
+                results = item
+    except Exception as e:
+        yield (
+            "❌ Evaluation failed",
+            f"❌ Evaluation failed: {str(e)}",
+            format_leaderboard_df(),
+        )
+        return
+    
+    yield status_update("⏳ Saving results...")
+    add_to_leaderboard(results)
+    
+    # Format results summary
+    summary = f"""## Results for {results.model_name}
+
+| Metric | Value |
+|--------|-------|
+| **MRR** | {results.mrr:.4f} |
+| **Recall@1** | {results.recall_at_1:.1%} |
+| **Recall@5** | {results.recall_at_5:.1%} |
+| **Recall@10** | {results.recall_at_10:.1%} |
+| **Bitext Accuracy** | {results.bitext_accuracy:.1%} |
+| **Avg True Pair Sim** | {results.avg_true_pair_similarity:.4f} |
+| **Avg Random Pair Sim** | {results.avg_random_pair_similarity:.4f} |
+| **Pairs Evaluated** | {results.num_pairs:,} |
+"""
+    
+    # Final yield with all results (clear status)
+    yield (
+        "✅ Complete!",
+        summary,
+        format_leaderboard_df(),
+    )
+
+
+def create_leaderboard_comparison():
+    """Create comparison chart of all models on leaderboard."""
+    load_leaderboard()
+    
+    if len(_leaderboard) < 2:
+        return None
+    
+    models = [e.get("model_name", e["model_id"]) for e in _leaderboard]
+    mrr = [e["mrr"] for e in _leaderboard]
+    r1 = [e["recall_at_1"] for e in _leaderboard]
+    r5 = [e["recall_at_5"] for e in _leaderboard]
+    r10 = [e["recall_at_10"] for e in _leaderboard]
+    bitext = [e["bitext_accuracy"] for e in _leaderboard]
+    
+    fig = go.Figure()
+    
+    fig.add_trace(go.Bar(name="MRR", x=models, y=mrr, marker_color="#2E86AB"))
+    fig.add_trace(go.Bar(name="R@1", x=models, y=r1, marker_color="#A23B72"))
+    fig.add_trace(go.Bar(name="R@5", x=models, y=r5, marker_color="#F18F01"))
+    fig.add_trace(go.Bar(name="R@10", x=models, y=r10, marker_color="#C73E1D"))
+    fig.add_trace(go.Bar(name="Bitext Acc", x=models, y=bitext, marker_color="#6B5B95"))
+    
+    fig.update_layout(
+        title="Model Comparison",
+        yaxis_title="Score",
+        yaxis_range=[0, 1],
+        barmode="group",
+        template="plotly_white",
+        height=400,
+    )
+    
+    return fig
+
+
+def update_model_inputs_visibility(choice):
+    """Show/hide custom model input and API key based on selection."""
+    show_custom = (choice == "custom")
+    show_api_key = requires_api_key(choice) if choice != "custom" else False
+    
+    # Update API key label based on model type
+    if show_api_key:
+        key_type = get_api_key_type(choice)
+        env_var = get_api_key_env_var(choice)
+        is_optional = api_key_optional(choice)
+        
+        if key_type == "voyage":
+            label = "Voyage AI API Key"
+            placeholder = f"Enter your Voyage AI API key (or set {env_var} env var)"
+        elif key_type == "gemini":
+            label = "Gemini API Key (optional if using gcloud)"
+            placeholder = f"Leave blank if using gcloud ADC, or enter API key / set {env_var}"
+        else:
+            label = "OpenAI API Key"
+            placeholder = f"Enter your OpenAI API key (or set {env_var} env var)"
+        return (
+            gr.update(visible=show_custom),
+            gr.update(visible=show_api_key, label=label, placeholder=placeholder),
+        )
+    
+    return (
+        gr.update(visible=show_custom),
+        gr.update(visible=show_api_key),
+    )
+
+
+# Build the Gradio interface
+def create_app():
+    """Create and return the Gradio app."""
+    
+    # Get all model choices - local models first, then API models
+    model_choices = []
+    
+    # Local models
+    for model_id, info in CURATED_MODELS.items():
+        model_choices.append((f"🖥️ {info['name']}", model_id))
+    
+    # API models
+    for model_id, info in API_MODELS.items():
+        model_choices.append((f"🌐 {info['name']}", model_id))
+    
+    # Custom option
+    model_choices.append(("⚙️ Custom Model (enter ID below)", "custom"))
+    
+    # Load initial data
+    load_benchmark()
+    load_leaderboard()
+    benchmark_stats = get_benchmark_stats(_benchmark_data) if _benchmark_data else {}
+    
+    with gr.Blocks(
+        title="Rabbinic Embedding Benchmark",
+        theme=gr.themes.Soft(
+            primary_hue="blue",
+            secondary_hue="orange",
+            font=gr.themes.GoogleFont("Source Sans Pro"),
+        ),
+        css="""
+        .main-header {
+            text-align: center;
+            margin-bottom: 1rem;
+        }
+        .stats-box {
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            color: white;
+            padding: 1rem;
+            border-radius: 8px;
+            margin: 0.5rem 0;
+        }
+        """,
+    ) as app:
+        
+        gr.Markdown(
+            """
+            # 📚 Rabbinic Hebrew/Aramaic Embedding Benchmark
+            
+            Evaluate embedding models on cross-lingual retrieval between Hebrew/Aramaic 
+            source texts and their English translations from Sefaria.
+            
+            **How it works:** Given a Hebrew/Aramaic text, can the model find its correct 
+            English translation from a pool of candidates? Models that excel at this task 
+            produce high-quality embeddings for Rabbinic literature.
+            """,
+            elem_classes=["main-header"],
+        )
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown(f"""
+                ### 📊 Benchmark Stats
+                - **Total Pairs:** {benchmark_stats.get('total_pairs', 'N/A'):,}
+                - **Categories:** {len(benchmark_stats.get('categories', {}))}
+                - **Avg Hebrew Length:** {benchmark_stats.get('avg_he_length', 0):.0f} chars
+                """)
+            
+            with gr.Column(scale=1):
+                gr.Markdown("""
+                ### 📏 Metrics
+                - **MRR:** Mean Reciprocal Rank
+                - **R@k:** Recall at k (correct in top k)
+                - **Bitext Acc:** True vs random pair classification
+                """)
+        
+        gr.Markdown("---")
+        
+        with gr.Tabs():
+            with gr.TabItem("🔬 Evaluate Model"):
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        model_dropdown = gr.Dropdown(
+                            choices=model_choices,
+                            value=model_choices[0][1],
+                            label="Select Model",
+                            info="Choose a curated model or enter a custom Hugging Face model ID",
+                        )
+                        
+                        custom_model_input = gr.Textbox(
+                            label="Custom Model ID",
+                            placeholder="e.g., organization/model-name",
+                            visible=False,
+                        )
+                        
+                        api_key_input = gr.Textbox(
+                            label="API Key",
+                            placeholder="Enter your API key (or set appropriate env var)",
+                            type="password",
+                            visible=False,
+                            info="Required for API-based models (OpenAI, Voyage AI). Your key is not stored.",
+                        )
+                        
+                        total_pairs = benchmark_stats.get('total_pairs', 1000)
+                        max_pairs_slider = gr.Slider(
+                            minimum=100,
+                            maximum=total_pairs,
+                            value=total_pairs,
+                            step=100,
+                            label="Max Pairs to Evaluate",
+                            info="Use fewer pairs for faster evaluation",
+                        )
+                    
+                    with gr.Column(scale=3):
+                        evaluate_btn = gr.Button(
+                            "🚀 Run Evaluation",
+                            variant="primary",
+                            size="lg",
+                        )
+                        
+                        status_text = gr.Markdown("")
+                        
+                        results_markdown = gr.Markdown("")
+            
+            with gr.TabItem("🏆 Leaderboard"):
+                leaderboard_table = gr.Dataframe(
+                    value=format_leaderboard_df(),
+                    label="Model Rankings",
+                    interactive=False,
+                )
+                
+                refresh_btn = gr.Button("🔄 Refresh Leaderboard")
+                
+                comparison_plot = gr.Plot(label="Model Comparison")
+        
+        gr.Markdown("""
+        ---
+        ### About
+        
+        This benchmark evaluates embedding models for Rabbinic Hebrew and Aramaic texts using 
+        cross-lingual retrieval. 
+
+        All texts and translations sourced from [Sefaria](https://www.sefaria.org).
+        """)
+        
+        # Event handlers
+        model_dropdown.change(
+            fn=update_model_inputs_visibility,
+            inputs=[model_dropdown],
+            outputs=[custom_model_input, api_key_input],
+        )
+        
+        evaluate_btn.click(
+            fn=run_evaluation,
+            inputs=[model_dropdown, custom_model_input, api_key_input, max_pairs_slider],
+            outputs=[status_text, results_markdown, leaderboard_table],
+            show_progress="hidden",
+        )
+        
+        refresh_btn.click(
+            fn=lambda: (format_leaderboard_df(), create_leaderboard_comparison()),
+            outputs=[leaderboard_table, comparison_plot],
+        )
+    
+    return app
+
+
+# Main entry point
+if __name__ == "__main__":
+    app = create_app()
+    app.launch()
+

benchmark-stats.txt

@@ -0,0 +1,14 @@
+  Total pairs: 3,721
+  Categories:
+    - Halacha: 160
+    - Hasidic/Kabbalistic: 304
+    - Jerusalem Talmud: 520
+    - Midrash Rabbah: 400
+    - Mishnah: 789
+    - Mussar/Ethics: 108
+    - Philosophy: 240
+    - Talmud: 480
+    - Tanakh Commentary: 680
+    - Targum: 40
+  Average Hebrew text length: 650 chars
+  Average English text length: 995 chars

build_benchmark.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""
+Script to build the benchmark dataset from Sefaria API.
+
+Run this script to fetch and cache parallel Hebrew/Aramaic-English text pairs
+from Sefaria for use in the embedding evaluation benchmark.
+
+Usage:
+    python build_benchmark.py [--max-per-text N] [--total N] [--output PATH]
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import requests
+
+from data_loader import (
+    build_benchmark_dataset,
+    get_benchmark_stats,
+    get_index_from_sefaria,
+    set_sefaria_host,
+    get_sefaria_host,
+    BENCHMARK_TEXTS,
+)
+
+
+def get_name_suggestions(title: str, host: str, limit: int = 5) -> list[str]:
+    """Get name suggestions from the Sefaria name API."""
+    try:
+        url = f"{host}/api/name/{title}"
+        response = requests.get(url, params={"limit": limit, "type": "ref"}, timeout=10)
+        if response.status_code == 200:
+            data = response.json()
+            # Return completions that are refs (book titles)
+            completions = data.get("completions", [])
+            return completions[:limit]
+    except requests.RequestException:
+        pass
+    return []
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build Rabbinic embedding benchmark dataset from Sefaria"
+    )
+    parser.add_argument(
+        "--max-per-text",
+        type=int,
+        default=40,
+        help="Maximum segments per text (default: 40)",
+    )
+    parser.add_argument(
+        "--total",
+        type=int,
+        default=10000,
+        help="Total target segments (default: 10000)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="benchmark_data/benchmark.json",
+        help="Output file path (default: benchmark_data/benchmark.json)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be fetched without making API calls",
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default=None,
+        help="Sefaria host URL (default: https://www.sefaria.org, or SEFARIA_HOST env var)",
+    )
+    parser.add_argument(
+        "--check-titles",
+        action="store_true",
+        help="Check all text titles against the API to verify they exist",
+    )
+    
+    args = parser.parse_args()
+    
+    # Configure Sefaria host if specified
+    if args.host:
+        set_sefaria_host(args.host)
+    
+    if args.check_titles:
+        print("="*60)
+        print("Checking Text Titles Against API")
+        print("="*60)
+        host = get_sefaria_host()
+        print(f"\nSefaria host: {host}\n")
+        
+        valid = []
+        invalid = []
+        suggestions = {}
+        
+        for category_key, category_info in BENCHMARK_TEXTS.items():
+            category_name = category_info["category"]
+            print(f"\n{category_name}:")
+            
+            for text in category_info["texts"]:
+                index = get_index_from_sefaria(text)
+                if index:
+                    print(f"  ✓ {text}")
+                    valid.append(text)
+                else:
+                    # Get suggestions from name API
+                    suggested = get_name_suggestions(text, host)
+                    suggestions[text] = suggested
+                    if suggested:
+                        print(f"  ✗ {text}  →  Did you mean: {suggested[0]}?")
+                    else:
+                        print(f"  ✗ {text}")
+                    invalid.append(text)
+        
+        print("\n" + "="*60)
+        print("SUMMARY")
+        print("="*60)
+        print(f"\nValid titles: {len(valid)}")
+        print(f"Invalid titles: {len(invalid)}")
+        
+        if invalid:
+            print(f"\nInvalid titles that need fixing:")
+            for title in invalid:
+                suggested = suggestions.get(title, [])
+                if suggested:
+                    print(f"  - {title}")
+                    print(f"    Suggestions: {', '.join(suggested[:3])}")
+                else:
+                    print(f"  - {title}  (no suggestions found)")
+        else:
+            print("\nAll titles are valid!")
+        
+        return
+    
+    if args.dry_run:
+        print("DRY RUN: Would fetch from these texts:\n")
+        print(f"Sefaria host: {get_sefaria_host()}")
+        total_texts = 0
+        for category_key, category_info in BENCHMARK_TEXTS.items():
+            print(f"\n{category_info['category']} ({category_info['language']}):")
+            for text in category_info["texts"]:
+                print(f"  - {text}")
+                total_texts += 1
+        print(f"\nTotal texts: {total_texts}")
+        print(f"Target segments per text: {args.max_per_text}")
+        print(f"Total target segments: {args.total}")
+        return
+    
+    print("="*60)
+    print("Building Rabbinic Embedding Benchmark Dataset")
+    print("="*60)
+    print(f"\nSettings:")
+    print(f"  Sefaria host: {get_sefaria_host()}")
+    print(f"  Max segments per text: {args.max_per_text}")
+    print(f"  Total target: {args.total}")
+    print(f"  Output: {args.output}")
+    print()
+    
+    # Ensure output directory exists
+    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+    
+    # Build the dataset
+    pairs = build_benchmark_dataset(
+        output_path=args.output,
+        segments_per_text=args.max_per_text,
+        total_target=args.total,
+    )
+    
+    # Print final statistics
+    stats = get_benchmark_stats(pairs)
+    
+    print("\n" + "="*60)
+    print("BENCHMARK COMPLETE")
+    print("="*60)
+    print(f"\nFinal Statistics:")
+    print(f"  Total pairs: {stats['total_pairs']:,}")
+    print(f"  Categories:")
+    for cat, count in sorted(stats["categories"].items()):
+        print(f"    - {cat}: {count:,}")
+    print(f"  Average Hebrew text length: {stats['avg_he_length']:.0f} chars")
+    print(f"  Average English text length: {stats['avg_en_length']:.0f} chars")
+    print(f"\nSaved to: {args.output}")
+
+
+if __name__ == "__main__":
+    main()
+

check_token_limits.py

@@ -0,0 +1,306 @@
+"""
+Check token limits for benchmark data entries.
+
+This script scans the benchmark dataset and flags entries that exceed
+the 8192 token limit used by OpenAI embedding models (text-embedding-ada-002,
+text-embedding-3-small, text-embedding-3-large).
+
+Uses tiktoken with the cl100k_base encoding, which is the tokenizer used
+by OpenAI's embedding models.
+"""
+
+import json
+import argparse
+from pathlib import Path
+from dataclasses import dataclass
+
+import tiktoken
+
+
+# OpenAI embedding models use cl100k_base encoding
+ENCODING_NAME = "cl100k_base"
+MAX_TOKENS = 8192
+
+
+@dataclass
+class TokenOverage:
+    """Represents an entry that exceeds the token limit."""
+    ref: str
+    category: str
+    field: str  # 'he', 'en', or 'combined'
+    token_count: int
+    char_count: int
+    text_preview: str  # First N characters of the text
+
+
+def count_tokens(text: str, encoding: tiktoken.Encoding) -> int:
+    """Count the number of tokens in a text string."""
+    return len(encoding.encode(text))
+
+
+def check_entry(
+    entry: dict,
+    encoding: tiktoken.Encoding,
+    max_tokens: int = MAX_TOKENS,
+    preview_length: int = 100
+) -> list[TokenOverage]:
+    """
+    Check a single entry for token limit violations.
+    
+    Args:
+        entry: Dictionary with 'ref', 'he', 'en', 'category' keys
+        encoding: tiktoken encoding to use
+        max_tokens: Maximum allowed tokens
+        preview_length: Number of characters to include in preview
+        
+    Returns:
+        List of TokenOverage objects for any fields exceeding the limit
+    """
+    overages = []
+    
+    ref = entry.get("ref", "unknown")
+    category = entry.get("category", "unknown")
+    
+    for field in ["he", "en"]:
+        text = entry.get(field, "")
+        if not text:
+            continue
+            
+        token_count = count_tokens(text, encoding)
+        
+        if token_count > max_tokens:
+            preview = text[:preview_length] + "..." if len(text) > preview_length else text
+            overages.append(TokenOverage(
+                ref=ref,
+                category=category,
+                field=field,
+                token_count=token_count,
+                char_count=len(text),
+                text_preview=preview
+            ))
+    
+    return overages
+
+
+def check_benchmark_data(
+    data_path: str,
+    max_tokens: int = MAX_TOKENS,
+    verbose: bool = False
+) -> tuple[list[TokenOverage], dict]:
+    """
+    Check all entries in the benchmark dataset for token limit violations.
+    
+    Args:
+        data_path: Path to the benchmark JSON file
+        max_tokens: Maximum allowed tokens (default: 8192)
+        verbose: Print progress information
+        
+    Returns:
+        Tuple of (list of overages, statistics dict)
+    """
+    # Load the encoding
+    if verbose:
+        print(f"Loading tokenizer: {ENCODING_NAME}")
+    encoding = tiktoken.get_encoding(ENCODING_NAME)
+    
+    # Load the data
+    if verbose:
+        print(f"Loading data from: {data_path}")
+    with open(data_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    
+    if verbose:
+        print(f"Checking {len(data)} entries for token limit ({max_tokens} tokens)...")
+    
+    # Check all entries
+    all_overages = []
+    token_counts_he = []
+    token_counts_en = []
+    
+    for i, entry in enumerate(data):
+        if verbose and (i + 1) % 1000 == 0:
+            print(f"  Processed {i + 1}/{len(data)} entries...")
+        
+        # Count tokens for statistics
+        he_text = entry.get("he", "")
+        en_text = entry.get("en", "")
+        
+        if he_text:
+            token_counts_he.append(count_tokens(he_text, encoding))
+        if en_text:
+            token_counts_en.append(count_tokens(en_text, encoding))
+        
+        # Check for overages
+        overages = check_entry(entry, encoding, max_tokens)
+        all_overages.extend(overages)
+    
+    # Compute statistics
+    stats = {
+        "total_entries": len(data),
+        "entries_with_overages": len(set(o.ref for o in all_overages)),
+        "total_overages": len(all_overages),
+        "he_overages": len([o for o in all_overages if o.field == "he"]),
+        "en_overages": len([o for o in all_overages if o.field == "en"]),
+        "max_tokens_checked": max_tokens,
+    }
+    
+    if token_counts_he:
+        stats["he_token_stats"] = {
+            "min": min(token_counts_he),
+            "max": max(token_counts_he),
+            "avg": sum(token_counts_he) / len(token_counts_he),
+            "total_entries": len(token_counts_he),
+        }
+    
+    if token_counts_en:
+        stats["en_token_stats"] = {
+            "min": min(token_counts_en),
+            "max": max(token_counts_en),
+            "avg": sum(token_counts_en) / len(token_counts_en),
+            "total_entries": len(token_counts_en),
+        }
+    
+    return all_overages, stats
+
+
+def print_report(overages: list[TokenOverage], stats: dict) -> None:
+    """Print a formatted report of token limit violations."""
+    print("\n" + "=" * 70)
+    print("TOKEN LIMIT CHECK REPORT")
+    print("=" * 70)
+    
+    print(f"\nDataset Summary:")
+    print(f"  Total entries checked: {stats['total_entries']:,}")
+    print(f"  Token limit: {stats['max_tokens_checked']:,}")
+    
+    if "he_token_stats" in stats:
+        he_stats = stats["he_token_stats"]
+        print(f"\nHebrew/Aramaic Token Statistics:")
+        print(f"  Min tokens: {he_stats['min']:,}")
+        print(f"  Max tokens: {he_stats['max']:,}")
+        print(f"  Avg tokens: {he_stats['avg']:.1f}")
+    
+    if "en_token_stats" in stats:
+        en_stats = stats["en_token_stats"]
+        print(f"\nEnglish Token Statistics:")
+        print(f"  Min tokens: {en_stats['min']:,}")
+        print(f"  Max tokens: {en_stats['max']:,}")
+        print(f"  Avg tokens: {en_stats['avg']:.1f}")
+    
+    print(f"\nOverage Summary:")
+    print(f"  Entries exceeding limit: {stats['entries_with_overages']:,}")
+    print(f"  Total field overages: {stats['total_overages']:,}")
+    print(f"    - Hebrew/Aramaic fields: {stats['he_overages']:,}")
+    print(f"    - English fields: {stats['en_overages']:,}")
+    
+    if overages:
+        print("\n" + "-" * 70)
+        print("FLAGGED ENTRIES (exceeding token limit):")
+        print("-" * 70)
+        
+        # Group by category
+        by_category = {}
+        for overage in overages:
+            if overage.category not in by_category:
+                by_category[overage.category] = []
+            by_category[overage.category].append(overage)
+        
+        for category, category_overages in sorted(by_category.items()):
+            print(f"\n[{category}] - {len(category_overages)} overage(s)")
+            for overage in category_overages:
+                print(f"\n  Reference: {overage.ref}")
+                print(f"  Field: {overage.field}")
+                print(f"  Token count: {overage.token_count:,} (limit: {stats['max_tokens_checked']:,})")
+                print(f"  Character count: {overage.char_count:,}")
+                print(f"  Preview: {overage.text_preview}")
+    else:
+        print("\n✓ No entries exceed the token limit!")
+    
+    print("\n" + "=" * 70)
+
+
+def save_report(
+    overages: list[TokenOverage],
+    stats: dict,
+    output_path: str
+) -> None:
+    """Save the report to a JSON file."""
+    report = {
+        "stats": stats,
+        "overages": [
+            {
+                "ref": o.ref,
+                "category": o.category,
+                "field": o.field,
+                "token_count": o.token_count,
+                "char_count": o.char_count,
+                "text_preview": o.text_preview,
+            }
+            for o in overages
+        ]
+    }
+    
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+    
+    print(f"\nReport saved to: {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Check benchmark data for entries exceeding OpenAI embedding token limits."
+    )
+    parser.add_argument(
+        "--data",
+        "-d",
+        type=str,
+        default="benchmark_data/benchmark.json",
+        help="Path to the benchmark JSON file (default: benchmark_data/benchmark.json)"
+    )
+    parser.add_argument(
+        "--max-tokens",
+        "-m",
+        type=int,
+        default=MAX_TOKENS,
+        help=f"Maximum allowed tokens (default: {MAX_TOKENS})"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        help="Path to save JSON report (optional)"
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Print progress information"
+    )
+    
+    args = parser.parse_args()
+    
+    # Check if data file exists
+    if not Path(args.data).exists():
+        print(f"Error: Data file not found: {args.data}")
+        return 1
+    
+    # Run the check
+    overages, stats = check_benchmark_data(
+        args.data,
+        max_tokens=args.max_tokens,
+        verbose=args.verbose
+    )
+    
+    # Print report
+    print_report(overages, stats)
+    
+    # Save report if requested
+    if args.output:
+        save_report(overages, stats, args.output)
+    
+    # Return exit code based on whether overages were found
+    return 1 if overages else 0
+
+
+if __name__ == "__main__":
+    exit(main())

data_loader.py

@@ -0,0 +1,828 @@
+"""
+Data loader for Rabbinic Hebrew/Aramaic benchmark texts from Sefaria API.
+
+Fetches parallel Hebrew/Aramaic + English text pairs across diverse categories.
+"""
+
+import json
+import os
+import re
+import time
+from pathlib import Path
+from typing import Optional
+
+import requests
+import tiktoken
+
+# Token limit for OpenAI embedding models (text-embedding-ada-002, text-embedding-3-*)
+# Using cl100k_base encoding
+MAX_EMBEDDING_TOKENS = 8192
+_tokenizer = None
+
+
+def get_tokenizer() -> tiktoken.Encoding:
+    """Get or create the tiktoken encoder (cached for performance)."""
+    global _tokenizer
+    if _tokenizer is None:
+        _tokenizer = tiktoken.get_encoding("cl100k_base")
+    return _tokenizer
+
+
+def count_tokens(text: str) -> int:
+    """Count the number of tokens in a text string using OpenAI's tokenizer."""
+    return len(get_tokenizer().encode(text))
+
+# Sefaria host - configurable via environment variable
+# Default is the public Sefaria API
+DEFAULT_SEFARIA_HOST = "https://www.sefaria.org"
+SEFARIA_HOST = os.environ.get("SEFARIA_HOST", DEFAULT_SEFARIA_HOST)
+
+
+def set_sefaria_host(host: str) -> None:
+    """Set the Sefaria host URL (e.g., 'http://localhost:8000')."""
+    global SEFARIA_HOST
+    # Remove trailing slash if present
+    SEFARIA_HOST = host.rstrip("/")
+
+
+def get_sefaria_host() -> str:
+    """Get the current Sefaria host URL."""
+    return SEFARIA_HOST
+
+# Text categories with confirmed English translations
+BENCHMARK_TEXTS = {
+    "talmud_bavli": {
+        "category": "Talmud",
+        "language": "Aramaic/Hebrew",
+        "texts": [
+            "Berakhot",
+            "Pesachim",
+            "Yoma",
+            "Megillah",
+            "Chagigah",
+            "Ketubot",
+            "Gittin",
+            "Bava Metzia",
+            "Sanhedrin",
+            "Avodah Zarah",
+            "Chullin",
+            "Niddah",
+        ],
+    },
+    "talmud_yerushalmi": {
+        "category": "Jerusalem Talmud",
+        "language": "Aramaic/Hebrew",
+        "texts": [
+            "Jerusalem Talmud Berakhot",
+            "Jerusalem Talmud Kilayim",
+            "Jerusalem Talmud Terumot",
+            "Jerusalem Talmud Shabbat",
+            "Jerusalem Talmud Shekalim",
+            "Jerusalem Talmud Sukkah",
+            "Jerusalem Talmud Sotah",
+            "Jerusalem Talmud Nedarim",
+            "Jerusalem Talmud Kiddushin",
+            "Jerusalem Talmud Bava Kamma",
+            "Jerusalem Talmud Sanhedrin",
+            "Jerusalem Talmud Avodah Zarah",
+            "Jerusalem Talmud Niddah",
+        ],
+    },
+    "mishnah": {
+        "category": "Mishnah",
+        "language": "Rabbinic Hebrew",
+        "texts": [
+            "Mishnah Berakhot",
+            "Mishnah Peah",
+            "Mishnah Kilayim",
+            "Mishnah Shabbat",
+            "Mishnah Pesachim",
+            "Mishnah Sukkah",
+            "Mishnah Taanit",
+            "Mishnah Chagigah",
+            "Mishnah Yevamot",
+            "Mishnah Sotah",
+            "Mishnah Kiddushin",
+            "Mishnah Bava Kamma",
+            "Mishnah Sanhedrin",
+            "Mishnah Eduyot",
+            "Mishnah Avot",
+            "Mishnah Zevachim",
+            "Mishnah Chullin",
+            "Mishnah Tamid",
+            "Mishnah Kelim",
+            "Mishnah Parah",
+            "Mishnah Niddah",
+        ],
+    },
+    "midrash_rabbah": {
+        "category": "Midrash Rabbah",
+        "language": "Hebrew/Aramaic",
+        "texts": [
+            "Bereishit Rabbah",
+            "Shemot Rabbah",
+            "Vayikra Rabbah",
+            "Bamidbar Rabbah",
+            "Devarim Rabbah",
+            "Shir HaShirim Rabbah",
+            "Ruth Rabbah",
+            "Eichah Rabbah",
+            "Kohelet Rabbah",
+            "Esther Rabbah",
+        ],
+    },
+    "tanakh_commentary": {
+        "category": "Tanakh Commentary",
+        "language": "Hebrew",
+        "texts": [
+            "Rashi on Genesis",
+            "Rashi on Exodus",
+            "Rashi on Leviticus",
+            "Rashi on Numbers",
+            "Rashi on Deuteronomy",
+            "Ramban on Genesis",
+            "Ramban on Exodus",
+            "Ramban on Leviticus",
+            "Ramban on Numbers",
+            "Ramban on Deuteronomy",
+            "Radak on Genesis",
+            "Akeidat Yitzchak",
+            "Rabbeinu Behaye, Bereshit",
+            "Rabbeinu Behaye, Shemot",
+            "Rabbeinu Behaye, Vayikra",
+            "Rabbeinu Behaye, Bamidbar",
+            "Rabbeinu Behaye, Devarim",
+        ],
+    },
+    "hasidic_kabbalistic": {
+        "category": "Hasidic/Kabbalistic",
+        "language": "Hebrew",
+        "texts": [
+            "Likutei Moharan",
+            "Tomer Devorah",
+            "Or Neerav, PART I",
+            "Or Neerav, PART II",
+            "Or Neerav, PART III",
+            "Shekel HaKodesh, On Abstinence",
+            "Shekel HaKodesh, On Wisdom",
+            "Kalach Pitchei Chokhmah",
+        ],
+    },
+    "halacha": {
+        "category": "Halacha",
+        "language": "Hebrew",
+        "texts": [
+            "Sefer HaChinukh",
+            "Shev Shmateta, Introduction",
+            "Mishneh Torah, Human Dispositions",
+            "Sefer Yesodei HaTorah",
+        ],
+    },
+    "philosophy": {
+        "category": "Philosophy",
+        "language": "Hebrew",
+        "texts": [
+            "Sefer HaIkkarim, Maamar 1",
+            "Sefer HaIkkarim, Maamar 2",
+            "Sefer HaIkkarim, Maamar 3",
+            "Guide for the Perplexed, Part 1",
+            "Guide for the Perplexed, Part 2",
+            "Guide for the Perplexed, Part 3",
+        ],
+    },
+    "targum": {
+        "category": "Targum",
+        "language": "Aramaic",
+        "texts": [
+            "Aramaic Targum to Song of Songs",
+        ],
+    },
+    "mussar": {
+        "category": "Mussar/Ethics",
+        "language": "Hebrew",
+        "texts": [
+            "Iggeret HaRamban",
+            "Shulchan Shel Arba",
+            "Chafetz Chaim",
+            "Yesod HaYirah, On Endurance",
+            "Yesod HaYirah, On Humility",
+            "Kav HaYashar",
+        ],
+    },
+}
+
+
+def strip_html(text: str) -> str:
+    """
+    Remove HTML tags from text.
+    
+    Some tags are dropped completely with their content:
+    - <sup class="footnote-marker">...</sup>
+    - <i class="footnote"...>...</i>
+    
+    Other tags are stripped but their inner content is preserved.
+    """
+    # First, remove footnote markers (simple, no nesting issues)
+    clean = re.sub(r'<sup[^>]*class="footnote-marker"[^>]*>.*?</sup>', '', text, flags=re.DOTALL)
+    
+    # Remove footnotes with nested <i> tags - need to handle nesting
+    # Strategy: find footnote start, then count <i> and </i> to find matching close
+    clean = _remove_footnote_tags(clean)
+    
+    # Then strip remaining HTML tags (keeping their content)
+    clean = re.sub(r"<[^>]+>", "", clean)
+    
+    # Clean up extra whitespace
+    clean = re.sub(r"\s+", " ", clean).strip()
+    return clean
+
+
+def _remove_footnote_tags(text: str) -> str:
+    """Remove <i class="footnote"...>...</i> tags, handling nested <i> tags."""
+    result = []
+    i = 0
+    
+    while i < len(text):
+        # Look for footnote opening tag
+        match = re.match(r'<i[^>]*class="footnote"[^>]*>', text[i:], flags=re.IGNORECASE)
+        if match:
+            # Found a footnote, now find the matching </i>
+            start = i + match.end()
+            depth = 1
+            j = start
+            
+            while j < len(text) and depth > 0:
+                if text[j:j+3].lower() == '<i ' or text[j:j+3].lower() == '<i>':
+                    depth += 1
+                    j += 1
+                elif text[j:j+4].lower() == '</i>':
+                    depth -= 1
+                    if depth == 0:
+                        # Skip past the closing </i>
+                        j += 4
+                        break
+                    j += 1
+                else:
+                    j += 1
+            
+            # Skip the entire footnote (from i to j)
+            i = j
+        else:
+            result.append(text[i])
+            i += 1
+    
+    return ''.join(result)
+
+
+def extract_bold_only(text: str) -> str:
+    """
+    Extract only content within <b>...</b> tags, for Talmud Bavli.
+    
+    The Steinsaltz English has bold for actual translation and non-bold for
+    elucidation. We only want the translation.
+    
+    Example:
+        "<b>The Rabbis say:</b> The time for... is <b>until midnight.</b>"
+        -> "The Rabbis say: until midnight."
+    """
+    # Find all content within <b>...</b> tags
+    bold_parts = re.findall(r'<b>(.*?)</b>', text, flags=re.DOTALL)
+    
+    if not bold_parts:
+        # No bold tags found, fall back to regular strip
+        return strip_html(text)
+    
+    # Strip any nested HTML from each bold part and join with spaces
+    cleaned_parts = [strip_html(part) for part in bold_parts]
+    
+    # Join parts, ensuring proper spacing
+    result = ' '.join(cleaned_parts)
+    
+    # Clean up extra whitespace
+    result = re.sub(r"\s+", " ", result).strip()
+    return result
+
+
+def get_text_from_sefaria(ref: str, retries: int = 3) -> Optional[dict]:
+    """
+    Fetch a text from Sefaria API.
+    
+    Args:
+        ref: Sefaria reference string (e.g., "Berakhot.2a")
+        retries: Number of retry attempts
+        
+    Returns:
+        Dict with 'he' (Hebrew/Aramaic) and 'en' (English) texts, or None if failed/error
+    """
+    url = f"{SEFARIA_HOST}/api/texts/{ref}"
+    params = {"context": 0}
+    
+    for attempt in range(retries):
+        try:
+            response = requests.get(url, params=params, timeout=30)
+            if response.status_code == 200:
+                data = response.json()
+                # Check if response contains an error
+                if "error" in data:
+                    return None
+                return data
+            elif response.status_code == 429:
+                # Rate limited, wait and retry
+                time.sleep(2 ** attempt)
+            else:
+                return None
+        except requests.RequestException:
+            if attempt < retries - 1:
+                time.sleep(1)
+            continue
+    return None
+
+
+def get_index_from_sefaria(title: str) -> Optional[dict]:
+    """
+    Get index/structure information for a text.
+    
+    Args:
+        title: The title of the text
+        
+    Returns:
+        Index data or None if failed or text not found
+    """
+    url = f"{SEFARIA_HOST}/api/index/{title}"
+    try:
+        response = requests.get(url, timeout=30)
+        if response.status_code == 200:
+            data = response.json()
+            # Check if response contains an error
+            if "error" in data:
+                return None
+            return data
+    except requests.RequestException:
+        pass
+    return None
+
+
+def extract_parallel_segments(data: dict, ref: str, category: str = "") -> list[dict]:
+    """
+    Extract parallel Hebrew/English segments from API response.
+    
+    Args:
+        data: API response data
+        ref: The reference string
+        category: Category name (used for special handling, e.g., "Talmud")
+        
+    Returns:
+        List of dicts with 'ref', 'he', 'en' keys
+    """
+    segments = []
+    
+    he_text = data.get("he", [])
+    en_text = data.get("text", [])
+    
+    # Handle nested arrays (common in Talmud)
+    if he_text and isinstance(he_text, list):
+        # Flatten if nested
+        if he_text and isinstance(he_text[0], list):
+            he_flat = []
+            en_flat = []
+            for i, (he_seg, en_seg) in enumerate(zip(he_text, en_text)):
+                if isinstance(he_seg, list):
+                    he_flat.extend(he_seg)
+                    en_flat.extend(en_seg if isinstance(en_seg, list) else [en_seg])
+                else:
+                    he_flat.append(he_seg)
+                    en_flat.append(en_seg)
+            he_text = he_flat
+            en_text = en_flat
+    
+    # Handle single string responses
+    if isinstance(he_text, str):
+        he_text = [he_text]
+    if isinstance(en_text, str):
+        en_text = [en_text]
+    
+    # For Talmud Bavli, extract only bold text (actual translation, not elucidation)
+    is_bavli = category == "Talmud"
+    
+    # Pair up segments
+    for i, (he, en) in enumerate(zip(he_text, en_text)):
+        if he and en:
+            he_clean = strip_html(str(he)) if he else ""
+            # Use bold-only extraction for Bavli English
+            if is_bavli:
+                en_clean = extract_bold_only(str(en)) if en else ""
+            else:
+                en_clean = strip_html(str(en)) if en else ""
+            
+            # Skip empty or very short segments
+            if len(he_clean) > 10 and len(en_clean) > 10:
+                # Check token limits for OpenAI embedding models
+                he_tokens = count_tokens(he_clean)
+                en_tokens = count_tokens(en_clean)
+                
+                if he_tokens > MAX_EMBEDDING_TOKENS:
+                    print(f"  Skipping {ref}:{i+1} - Hebrew text exceeds token limit ({he_tokens} > {MAX_EMBEDDING_TOKENS})")
+                    continue
+                if en_tokens > MAX_EMBEDDING_TOKENS:
+                    print(f"  Skipping {ref}:{i+1} - English text exceeds token limit ({en_tokens} > {MAX_EMBEDDING_TOKENS})")
+                    continue
+                
+                segments.append({
+                    "ref": f"{ref}:{i+1}" if ":" not in ref else ref,
+                    "he": he_clean,
+                    "en": en_clean,
+                })
+    
+    return segments
+
+
+def fetch_text_pairs(
+    text_title: str,
+    category: str,
+    max_segments: int = 500,
+    delay: float = 0.5
+) -> list[dict]:
+    """
+    Fetch parallel text pairs for a given text.
+    
+    Args:
+        text_title: Title of the text to fetch
+        category: Category name for metadata
+        max_segments: Maximum segments to fetch per text
+        delay: Delay between API calls (rate limiting)
+        
+    Returns:
+        List of segment dicts with ref, he, en, category
+    """
+    pairs = []
+    
+    # Get text index to understand structure
+    index = get_index_from_sefaria(text_title)
+    if not index:
+        print(f"  Could not get index for {text_title}")
+        return pairs
+    
+    # Determine refs to fetch based on text structure
+    schema = index.get("schema", {})
+    
+    # For simple texts, just fetch the whole thing
+    if schema.get("nodeType") == "JaggedArrayNode":
+        depth = schema.get("depth", 2)
+        address_types = schema.get("addressTypes", [])
+        
+        # Check if this uses Talmud daf notation (2a, 2b, etc.)
+        uses_talmud_daf = address_types and address_types[0] == "Talmud"
+        
+        if uses_talmud_daf:
+            # Talmud-style structure with daf notation (e.g., Berakhot.2a)
+            # Start from daf 3 for Jerusalem Talmud to avoid overlap with Bavli
+            start_daf = 3 if category == "Jerusalem Talmud" else 2
+            # Fetch daf by daf
+            done = False
+            for daf_num in range(start_daf, 200):
+                if len(pairs) >= max_segments or done:
+                    break
+                
+                for side in ["a", "b"]:
+                    if len(pairs) >= max_segments:
+                        break
+                        
+                    ref = f"{text_title}.{daf_num}{side}"
+                    data = get_text_from_sefaria(ref)
+                    
+                    # None means API error (daf doesn't exist)
+                    if data is None:
+                        if side == "a":
+                            done = True  # Daf doesn't exist, we're done with tractate
+                        break
+                    
+                    if not data.get("he"):
+                        continue  # Empty side, try next
+                        
+                    segments = extract_parallel_segments(data, ref, category)
+                    for seg in segments:
+                        seg["category"] = category
+                    pairs.extend(segments)
+                    
+                    time.sleep(delay)
+                
+        elif depth == 1:
+            # Single-level structure (e.g., Iggeret HaRamban - just paragraphs)
+            # Fetch the whole text at once
+            data = get_text_from_sefaria(text_title)
+            if data and data.get("he"):
+                segments = extract_parallel_segments(data, text_title, category)
+                for seg in segments:
+                    seg["category"] = category
+                pairs.extend(segments)
+                
+        elif depth == 2:
+            # Two-level structure (e.g., Mishnah chapter:verse)
+            # Start from chapter 2 for Mishnah to avoid overlap with Talmud
+            start_chapter = 2 if category == "Mishnah" else 1
+            consecutive_empty = 0
+            # Fetch chapter by chapter
+            for chapter in range(start_chapter, 200):  # Reasonable upper bound
+                if len(pairs) >= max_segments:
+                    break
+                    
+                ref = f"{text_title}.{chapter}"
+                data = get_text_from_sefaria(ref)
+                
+                # None means API error (ref doesn't exist)
+                if data is None:
+                    break
+                
+                # Empty array means chapter exists but has no content
+                if not data.get("he"):
+                    consecutive_empty += 1
+                    if consecutive_empty >= 5:
+                        break  # Probably past end of book
+                    time.sleep(delay)
+                    continue
+                    
+                consecutive_empty = 0
+                segments = extract_parallel_segments(data, ref, category)
+                for seg in segments:
+                    seg["category"] = category
+                pairs.extend(segments)
+                
+                time.sleep(delay)
+                
+        elif depth >= 3:
+            # Three+ level structure (e.g., commentary chapter:verse:comment)
+            # Fetch chapter.verse by chapter.verse
+            # For Jerusalem Talmud, start from 1.3 to avoid overlap with Bavli
+            start_verse = 3 if category == "Jerusalem Talmud" else 1
+            consecutive_empty_chapters = 0
+            for chapter in range(1, 200):
+                if len(pairs) >= max_segments:
+                    break
+                
+                chapter_had_content = False
+                # Use start_verse only for first chapter
+                first_verse = start_verse if chapter == 1 else 1
+                for verse in range(first_verse, 100):
+                    if len(pairs) >= max_segments:
+                        break
+                    
+                    ref = f"{text_title}.{chapter}.{verse}"
+                    data = get_text_from_sefaria(ref)
+                    
+                    # None means API error (ref doesn't exist)
+                    if data is None:
+                        break  # No more verses in this chapter
+                    
+                    # Empty array means verse exists but has no content
+                    if not data.get("he"):
+                        continue
+                        
+                    chapter_had_content = True
+                    segments = extract_parallel_segments(data, ref, category)
+                    for seg in segments:
+                        seg["category"] = category
+                    pairs.extend(segments)
+                    
+                    time.sleep(delay)
+                
+                if not chapter_had_content:
+                    consecutive_empty_chapters += 1
+                    if consecutive_empty_chapters >= 5:
+                        break  # Probably past end of book
+                else:
+                    consecutive_empty_chapters = 0
+    
+    else:
+        # Complex structure (SchemaNode) - try different ref patterns
+        # First try simple numeric refs (works for Sefer HaChinukh style)
+        consecutive_empty = 0
+        for section in range(1, 1000):
+            if len(pairs) >= max_segments:
+                break
+            
+            ref = f"{text_title}.{section}"
+            data = get_text_from_sefaria(ref)
+            
+            if data is None:
+                break
+            
+            if not data.get("he"):
+                consecutive_empty += 1
+                if consecutive_empty >= 5:
+                    break
+                time.sleep(delay)
+                continue
+                
+            consecutive_empty = 0
+            segments = extract_parallel_segments(data, ref, category)
+            for seg in segments:
+                seg["category"] = category
+            pairs.extend(segments)
+            
+            time.sleep(delay)
+        
+        # If we haven't reached max_segments, try chapter.verse style refs (commentary pattern)
+        if len(pairs) < max_segments:
+            consecutive_empty = 0
+            for chapter in range(1, 100):
+                if len(pairs) >= max_segments:
+                    break
+                
+                chapter_had_content = False
+                for verse in range(1, 50):
+                    if len(pairs) >= max_segments:
+                        break
+                    
+                    ref = f"{text_title}.{chapter}.{verse}"
+                    data = get_text_from_sefaria(ref)
+                    
+                    if data is None:
+                        break  # This verse doesn't exist, try next chapter
+                    
+                    if data.get("he"):
+                        chapter_had_content = True
+                        consecutive_empty = 0
+                        segments = extract_parallel_segments(data, ref, category)
+                        for seg in segments:
+                            seg["category"] = category
+                        pairs.extend(segments)
+                    
+                    time.sleep(delay)
+                
+                if not chapter_had_content:
+                    consecutive_empty += 1
+                    if consecutive_empty >= 5:
+                        break
+    
+    return pairs[:max_segments]
+
+
+def build_benchmark_dataset(
+    output_path: str = "benchmark_data/benchmark.json",
+    segments_per_text: int = 200,
+    total_target: int = 10000,
+) -> list[dict]:
+    """
+    Build the full benchmark dataset from all configured texts.
+    
+    Args:
+        output_path: Path to save the benchmark JSON
+        segments_per_text: Target segments per text
+        total_target: Overall target segment count
+        
+    Returns:
+        List of all benchmark pairs
+    """
+    all_pairs = []
+    
+    for category_key, category_info in BENCHMARK_TEXTS.items():
+        category_name = category_info["category"]
+        texts = category_info["texts"]
+        
+        print(f"\n{'='*60}")
+        print(f"Processing category: {category_name}")
+        print(f"{'='*60}")
+        
+        for text_title in texts:
+            if len(all_pairs) >= total_target:
+                break
+                
+            print(f"\nFetching: {text_title}")
+            
+            pairs = fetch_text_pairs(
+                text_title,
+                category_name,
+                max_segments=segments_per_text,
+            )
+            
+            print(f"  Got {len(pairs)} pairs")
+            all_pairs.extend(pairs)
+            
+        if len(all_pairs) >= total_target:
+            break
+    
+    # Save to file
+    output_file = Path(output_path)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(all_pairs, f, ensure_ascii=False, indent=2)
+    
+    print(f"\n{'='*60}")
+    print(f"Total pairs collected: {len(all_pairs)}")
+    print(f"Saved to: {output_path}")
+    
+    # Save stats to markdown file
+    stats = get_benchmark_stats(all_pairs)
+    save_stats_markdown(stats, output_path)
+    
+    return all_pairs
+
+
+def load_benchmark_dataset(path: str = "benchmark_data/benchmark.json") -> list[dict]:
+    """
+    Load the pre-cached benchmark dataset.
+    
+    Args:
+        path: Path to the benchmark JSON file
+        
+    Returns:
+        List of benchmark pairs
+    """
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_benchmark_stats(pairs: list[dict]) -> dict:
+    """
+    Get statistics about the benchmark dataset.
+    
+    Args:
+        pairs: List of benchmark pairs
+        
+    Returns:
+        Dict with category counts and other stats
+    """
+    from collections import Counter
+    
+    categories = Counter(p["category"] for p in pairs)
+    
+    he_lengths = [len(p["he"]) for p in pairs]
+    en_lengths = [len(p["en"]) for p in pairs]
+    
+    return {
+        "total_pairs": len(pairs),
+        "categories": dict(categories),
+        "avg_he_length": sum(he_lengths) / len(he_lengths) if he_lengths else 0,
+        "avg_en_length": sum(en_lengths) / len(en_lengths) if en_lengths else 0,
+    }
+
+
+def save_stats_markdown(stats: dict, data_path: str) -> str:
+    """
+    Save benchmark statistics to a markdown file alongside the data.
+    
+    Args:
+        stats: Statistics dict from get_benchmark_stats()
+        data_path: Path to the data file (used to derive stats file path)
+        
+    Returns:
+        Path to the saved markdown file
+    """
+    from datetime import datetime
+    
+    # Derive markdown path from data path
+    data_file = Path(data_path)
+    stats_path = data_file.with_suffix(".stats.md")
+    
+    # Build markdown content
+    lines = [
+        "# Benchmark Dataset Statistics",
+        "",
+        f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+        "",
+        "## Summary",
+        "",
+        f"- **Total pairs:** {stats['total_pairs']:,}",
+        f"- **Average Hebrew length:** {stats['avg_he_length']:.0f} chars",
+        f"- **Average English length:** {stats['avg_en_length']:.0f} chars",
+        "",
+        "## Category Breakdown",
+        "",
+        "| Category | Count |",
+        "|----------|-------|",
+    ]
+    
+    # Sort categories by count (descending)
+    sorted_categories = sorted(
+        stats["categories"].items(),
+        key=lambda x: x[1],
+        reverse=True
+    )
+    
+    for category, count in sorted_categories:
+        lines.append(f"| {category} | {count:,} |")
+    
+    lines.append("")
+    
+    # Write to file
+    with open(stats_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(lines))
+    
+    print(f"Stats saved to: {stats_path}")
+    return str(stats_path)
+
+
+if __name__ == "__main__":
+    # Build the benchmark dataset
+    print("Building Rabbinic Hebrew/Aramaic benchmark dataset...")
+    pairs = build_benchmark_dataset()
+    
+    # Print stats
+    stats = get_benchmark_stats(pairs)
+    print(f"\nDataset Statistics:")
+    print(f"  Total pairs: {stats['total_pairs']}")
+    print(f"  Categories: {stats['categories']}")
+    print(f"  Avg Hebrew length: {stats['avg_he_length']:.0f} chars")
+    print(f"  Avg English length: {stats['avg_en_length']:.0f} chars")
+

evaluation.py

@@ -0,0 +1,384 @@
+"""
+Cross-lingual retrieval evaluation for Rabbinic embedding benchmark.
+
+Computes retrieval metrics to measure how well embedding models align
+Hebrew/Aramaic source texts with their English translations.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+import numpy as np
+
+
+@dataclass
+class EvaluationResults:
+    """Container for evaluation results."""
+    
+    model_id: str
+    model_name: str
+    
+    # Core retrieval metrics
+    recall_at_1: float
+    recall_at_5: float
+    recall_at_10: float
+    mrr: float  # Mean Reciprocal Rank
+    
+    # Additional metrics
+    bitext_accuracy: float  # True pair vs random pair classification
+    avg_true_pair_similarity: float
+    avg_random_pair_similarity: float
+    
+    # Metadata
+    num_pairs: int
+    categories: dict[str, int]
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "model_id": self.model_id,
+            "model_name": self.model_name,
+            "recall_at_1": self.recall_at_1,
+            "recall_at_5": self.recall_at_5,
+            "recall_at_10": self.recall_at_10,
+            "mrr": self.mrr,
+            "bitext_accuracy": self.bitext_accuracy,
+            "avg_true_pair_similarity": self.avg_true_pair_similarity,
+            "avg_random_pair_similarity": self.avg_random_pair_similarity,
+            "num_pairs": self.num_pairs,
+            "categories": self.categories,
+        }
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "EvaluationResults":
+        """Create from dictionary."""
+        return cls(**data)
+
+
+def compute_similarity_matrix(
+    query_embeddings: np.ndarray,
+    passage_embeddings: np.ndarray,
+) -> np.ndarray:
+    """
+    Compute cosine similarity matrix between queries and passages.
+    
+    Assumes embeddings are already L2-normalized.
+    
+    Args:
+        query_embeddings: (N, D) array of query embeddings
+        passage_embeddings: (M, D) array of passage embeddings
+        
+    Returns:
+        (N, M) similarity matrix
+    """
+    return np.dot(query_embeddings, passage_embeddings.T)
+
+
+def compute_retrieval_metrics(
+    similarity_matrix: np.ndarray,
+    k_values: list[int] = [1, 5, 10],
+) -> dict[str, float]:
+    """
+    Compute retrieval metrics from similarity matrix.
+    
+    Assumes the correct match for query i is passage i (diagonal).
+    
+    Args:
+        similarity_matrix: (N, N) similarity matrix where diagonal is true matches
+        k_values: List of k values for Recall@k
+        
+    Returns:
+        Dict with recall@k and mrr values
+    """
+    n = similarity_matrix.shape[0]
+    
+    # Get rankings for each query
+    # Negate to sort descending (highest similarity first)
+    rankings = np.argsort(-similarity_matrix, axis=1)
+    
+    # Find rank of true match (diagonal) for each query
+    true_ranks = np.zeros(n, dtype=int)
+    for i in range(n):
+        # Find position of index i in the ranking for query i
+        true_ranks[i] = np.where(rankings[i] == i)[0][0]
+    
+    results = {}
+    
+    # Recall@k: fraction where true match is in top k
+    for k in k_values:
+        recall = np.mean(true_ranks < k)
+        results[f"recall_at_{k}"] = float(recall)
+    
+    # MRR: Mean Reciprocal Rank
+    reciprocal_ranks = 1.0 / (true_ranks + 1)  # +1 because ranks are 0-indexed
+    results["mrr"] = float(np.mean(reciprocal_ranks))
+    
+    return results
+
+
+def compute_bitext_accuracy(
+    similarity_matrix: np.ndarray,
+    num_negatives: int = 10,
+) -> tuple[float, float, float]:
+    """
+    Compute bitext mining accuracy.
+    
+    For each true pair, sample random negative pairs and check if the model
+    correctly ranks the true pair higher.
+    
+    Args:
+        similarity_matrix: (N, N) similarity matrix
+        num_negatives: Number of negative samples per true pair
+        
+    Returns:
+        Tuple of (accuracy, avg_true_sim, avg_random_sim)
+    """
+    n = similarity_matrix.shape[0]
+    
+    # True pair similarities (diagonal)
+    true_similarities = np.diag(similarity_matrix)
+    
+    # Sample random negative pairs
+    correct = 0
+    total = 0
+    random_sims = []
+    
+    rng = np.random.default_rng(42)
+    
+    for i in range(n):
+        true_sim = true_similarities[i]
+        
+        # Sample random passage indices (not the true match)
+        neg_indices = rng.choice(
+            [j for j in range(n) if j != i],
+            size=min(num_negatives, n - 1),
+            replace=False,
+        )
+        
+        for j in neg_indices:
+            neg_sim = similarity_matrix[i, j]
+            random_sims.append(neg_sim)
+            
+            if true_sim > neg_sim:
+                correct += 1
+            total += 1
+    
+    accuracy = correct / total if total > 0 else 0.0
+    avg_true = float(np.mean(true_similarities))
+    avg_random = float(np.mean(random_sims)) if random_sims else 0.0
+    
+    return accuracy, avg_true, avg_random
+
+
+def evaluate_model(
+    model,
+    benchmark_pairs: list[dict],
+    batch_size: int = 32,
+    max_pairs: Optional[int] = None,
+    progress_callback=None,
+) -> EvaluationResults:
+    """
+    Run full evaluation of a model on the benchmark.
+    
+    Args:
+        model: EmbeddingModel instance
+        benchmark_pairs: List of benchmark pairs with 'he', 'en', 'category' keys
+        batch_size: Batch size for encoding
+        max_pairs: Maximum pairs to evaluate (for faster testing)
+        progress_callback: Optional callback for progress updates
+        
+    Returns:
+        EvaluationResults with all metrics
+    """
+    # Use streaming version and return final result
+    result = None
+    for item in evaluate_model_streaming(model, benchmark_pairs, batch_size, max_pairs):
+        if isinstance(item, str):
+            if progress_callback:
+                progress_callback(0.5, item)
+        else:
+            result = item
+    return result
+
+
+def evaluate_model_streaming(
+    model,
+    benchmark_pairs: list[dict],
+    batch_size: int = 32,
+    max_pairs: Optional[int] = None,
+):
+    """
+    Run evaluation with streaming progress updates.
+    
+    Yields progress strings during encoding, then yields final EvaluationResults.
+    
+    Args:
+        model: EmbeddingModel instance
+        benchmark_pairs: List of benchmark pairs with 'he', 'en', 'category' keys
+        batch_size: Batch size for encoding
+        max_pairs: Maximum pairs to evaluate (for faster testing)
+        
+    Yields:
+        Progress strings, then final EvaluationResults
+    """
+    from collections import Counter
+    
+    # Optionally limit pairs
+    if max_pairs and len(benchmark_pairs) > max_pairs:
+        benchmark_pairs = benchmark_pairs[:max_pairs]
+    
+    # Extract texts
+    he_texts = [p["he"] for p in benchmark_pairs]
+    en_texts = [p["en"] for p in benchmark_pairs]
+    categories = Counter(p.get("category", "Unknown") for p in benchmark_pairs)
+    n_total = len(he_texts)
+    
+    # Encode Hebrew texts in batches with progress
+    yield f"⏳ Encoding Hebrew/Aramaic texts: 0/{n_total:,}"
+    he_embeddings_list = []
+    for i in range(0, len(he_texts), batch_size):
+        batch = he_texts[i:i + batch_size]
+        batch_emb = model.encode(
+            batch,
+            is_query=True,
+            batch_size=batch_size,
+            show_progress=False,
+        )
+        he_embeddings_list.append(batch_emb)
+        done = min(i + batch_size, len(he_texts))
+        yield f"⏳ Encoding Hebrew/Aramaic texts: {done:,}/{n_total:,}"
+    
+    he_embeddings = np.vstack(he_embeddings_list)
+    
+    # Encode English texts in batches with progress
+    yield f"⏳ Encoding English texts: 0/{n_total:,}"
+    en_embeddings_list = []
+    for i in range(0, len(en_texts), batch_size):
+        batch = en_texts[i:i + batch_size]
+        batch_emb = model.encode(
+            batch,
+            is_query=False,
+            batch_size=batch_size,
+            show_progress=False,
+        )
+        en_embeddings_list.append(batch_emb)
+        done = min(i + batch_size, len(en_texts))
+        yield f"⏳ Encoding English texts: {done:,}/{n_total:,}"
+    
+    en_embeddings = np.vstack(en_embeddings_list)
+    
+    yield "⏳ Computing similarity matrix..."
+    similarity_matrix = compute_similarity_matrix(he_embeddings, en_embeddings)
+    
+    yield "⏳ Computing retrieval metrics..."
+    retrieval_metrics = compute_retrieval_metrics(similarity_matrix)
+    
+    yield "⏳ Computing bitext accuracy..."
+    bitext_acc, avg_true_sim, avg_random_sim = compute_bitext_accuracy(
+        similarity_matrix
+    )
+    
+    # Yield final results
+    yield EvaluationResults(
+        model_id=model.model_id,
+        model_name=model.name,
+        recall_at_1=retrieval_metrics["recall_at_1"],
+        recall_at_5=retrieval_metrics["recall_at_5"],
+        recall_at_10=retrieval_metrics["recall_at_10"],
+        mrr=retrieval_metrics["mrr"],
+        bitext_accuracy=bitext_acc,
+        avg_true_pair_similarity=avg_true_sim,
+        avg_random_pair_similarity=avg_random_sim,
+        num_pairs=len(benchmark_pairs),
+        categories=dict(categories),
+    )
+
+
+def evaluate_by_category(
+    model,
+    benchmark_pairs: list[dict],
+    batch_size: int = 32,
+) -> dict[str, EvaluationResults]:
+    """
+    Run evaluation broken down by category.
+    
+    Args:
+        model: EmbeddingModel instance
+        benchmark_pairs: List of benchmark pairs
+        batch_size: Batch size for encoding
+        
+    Returns:
+        Dict mapping category name to EvaluationResults
+    """
+    from collections import defaultdict
+    
+    # Group pairs by category
+    by_category = defaultdict(list)
+    for pair in benchmark_pairs:
+        category = pair.get("category", "Unknown")
+        by_category[category].append(pair)
+    
+    results = {}
+    for category, pairs in by_category.items():
+        print(f"Evaluating category: {category} ({len(pairs)} pairs)")
+        results[category] = evaluate_model(model, pairs, batch_size=batch_size)
+    
+    return results
+
+
+def get_rank_distribution(
+    similarity_matrix: np.ndarray,
+    bins: list[int] = [1, 5, 10, 50, 100],
+) -> dict[str, int]:
+    """
+    Get distribution of true match ranks.
+    
+    Args:
+        similarity_matrix: (N, N) similarity matrix
+        bins: Bin boundaries for histogram
+        
+    Returns:
+        Dict mapping bin labels to counts
+    """
+    n = similarity_matrix.shape[0]
+    rankings = np.argsort(-similarity_matrix, axis=1)
+    
+    # Find true rank for each query
+    true_ranks = np.zeros(n, dtype=int)
+    for i in range(n):
+        true_ranks[i] = np.where(rankings[i] == i)[0][0]
+    
+    # Create histogram
+    distribution = {}
+    prev_bin = 0
+    for bin_edge in bins:
+        count = np.sum((true_ranks >= prev_bin) & (true_ranks < bin_edge))
+        label = f"{prev_bin+1}-{bin_edge}" if prev_bin > 0 else f"Top {bin_edge}"
+        distribution[label] = int(count)
+        prev_bin = bin_edge
+    
+    # Count remaining
+    remaining = np.sum(true_ranks >= bins[-1])
+    distribution[f">{bins[-1]}"] = int(remaining)
+    
+    return distribution
+
+
+if __name__ == "__main__":
+    # Test with sample data
+    print("Testing evaluation functions...")
+    
+    # Create sample similarity matrix (perfect retrieval)
+    n = 100
+    perfect_matrix = np.eye(n) + np.random.randn(n, n) * 0.1
+    
+    metrics = compute_retrieval_metrics(perfect_matrix)
+    print(f"Perfect retrieval metrics: {metrics}")
+    
+    # Test with random matrix
+    random_matrix = np.random.randn(n, n)
+    random_matrix = random_matrix / np.linalg.norm(random_matrix, axis=1, keepdims=True)
+    random_matrix = np.dot(random_matrix, random_matrix.T)
+    
+    metrics = compute_retrieval_metrics(random_matrix)
+    print(f"Random retrieval metrics: {metrics}")
+

models.py

@@ -0,0 +1,1063 @@
+"""
+Model loading and embedding interface for the Rabbinic embedding benchmark.
+
+Supports:
+- Curated models from Hugging Face (sentence-transformers)
+- Any Hugging Face sentence-transformer model
+- API-based models (OpenAI, Voyage AI, Google Gemini)
+"""
+
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+import numpy as np
+
+# Curated local models known to work well for multilingual tasks
+CURATED_MODELS = {
+    "intfloat/multilingual-e5-large": {
+        "name": "Multilingual E5 Large",
+        "description": "Strong multilingual model from Microsoft, 560M params",
+        "type": "local",
+        "query_prefix": "query: ",
+        "passage_prefix": "passage: ",
+    },
+    "intfloat/multilingual-e5-base": {
+        "name": "Multilingual E5 Base",
+        "description": "Smaller multilingual E5, 278M params",
+        "type": "local",
+        "query_prefix": "query: ",
+        "passage_prefix": "passage: ",
+    },
+    "sentence-transformers/paraphrase-multilingual-mpnet-base-v2": {
+        "name": "Multilingual MPNet",
+        "description": "Classic multilingual sentence transformer, 278M params",
+        "type": "local",
+        "query_prefix": "",
+        "passage_prefix": "",
+    },
+    "BAAI/bge-m3": {
+        "name": "BGE-M3",
+        "description": "Multi-lingual, multi-functionality, multi-granularity model from BAAI",
+        "type": "local",
+        "query_prefix": "",
+        "passage_prefix": "",
+    },
+    "intfloat/e5-mistral-7b-instruct": {
+        "name": "E5 Mistral 7B",
+        "description": "Large instruction-tuned embedding model, 7B params (requires GPU)",
+        "type": "local",
+        "query_prefix": "Instruct: Retrieve semantically similar text\nQuery: ",
+        "passage_prefix": "",
+    },
+    "Alibaba-NLP/gte-multilingual-base": {
+        "name": "GTE Multilingual Base",
+        "description": "General Text Embeddings multilingual model from Alibaba",
+        "type": "local",
+        "query_prefix": "",
+        "passage_prefix": "",
+    },
+}
+
+# API-based models
+API_MODELS = {
+    "openai/text-embedding-3-large": {
+        "name": "OpenAI text-embedding-3-large",
+        "description": "OpenAI's best embedding model, 3072 dimensions (API key required)",
+        "type": "openai",
+        "model_name": "text-embedding-3-large",
+        "dimensions": 3072,
+    },
+    "openai/text-embedding-3-small": {
+        "name": "OpenAI text-embedding-3-small",
+        "description": "OpenAI's efficient embedding model, 1536 dimensions (API key required)",
+        "type": "openai",
+        "model_name": "text-embedding-3-small",
+        "dimensions": 1536,
+    },
+    "openai/text-embedding-ada-002": {
+        "name": "OpenAI Ada 002",
+        "description": "OpenAI's legacy embedding model, 1536 dimensions (API key required)",
+        "type": "openai",
+        "model_name": "text-embedding-ada-002",
+        "dimensions": 1536,
+    },
+    "voyage/voyage-3.5": {
+        "name": "Voyage AI voyage-3.5",
+        "description": "Voyage AI's latest embedding model (API key required)",
+        "type": "voyage",
+        "model_name": "voyage-3.5",
+        "dimensions": 1024,
+    },
+    "voyage/voyage-3.5-lite": {
+        "name": "Voyage AI voyage-3.5-lite",
+        "description": "Voyage AI's efficient embedding model (API key required)",
+        "type": "voyage",
+        "model_name": "voyage-3.5-lite",
+        "dimensions": 1024,
+    },
+    "voyage/voyage-3": {
+        "name": "Voyage AI voyage-3",
+        "description": "Voyage AI's general purpose embedding model (API key required)",
+        "type": "voyage",
+        "model_name": "voyage-3",
+        "dimensions": 1024,
+    },
+    "voyage/voyage-3-lite": {
+        "name": "Voyage AI voyage-3-lite",
+        "description": "Voyage AI's lightweight embedding model (API key required)",
+        "type": "voyage",
+        "model_name": "voyage-3-lite",
+        "dimensions": 512,
+    },
+    "voyage/voyage-multilingual-2": {
+        "name": "Voyage AI voyage-multilingual-2",
+        "description": "Voyage AI's multilingual embedding model, optimized for non-English (API key required)",
+        "type": "voyage",
+        "model_name": "voyage-multilingual-2",
+        "dimensions": 1024,
+    },
+    "gemini/gemini-embedding-001": {
+        "name": "Gemini Embedding 001",
+        "description": "Google's Gemini embedding model, 3072 dimensions (API key required)",
+        "type": "gemini",
+        "model_name": "gemini-embedding-001",
+        "dimensions": 3072,
+    },
+    "gemini/gemini-embedding-001-768": {
+        "name": "Gemini Embedding 001 (768d)",
+        "description": "Google's Gemini embedding model, 768 dimensions (API key required)",
+        "type": "gemini",
+        "model_name": "gemini-embedding-001",
+        "dimensions": 768,
+    },
+    "gemini/gemini-embedding-001-1536": {
+        "name": "Gemini Embedding 001 (1536d)",
+        "description": "Google's Gemini embedding model, 1536 dimensions (API key required)",
+        "type": "gemini",
+        "model_name": "gemini-embedding-001",
+        "dimensions": 1536,
+    },
+}
+
+# Merge all models for easy lookup
+ALL_MODELS = {**CURATED_MODELS, **API_MODELS}
+
+
+class BaseEmbeddingModel(ABC):
+    """Abstract base class for embedding models."""
+    
+    model_id: str
+    embedding_dim: int
+    
+    @abstractmethod
+    def encode(
+        self,
+        texts: list[str],
+        is_query: bool = False,
+        batch_size: int = 32,
+        show_progress: bool = True,
+        normalize: bool = True,
+    ) -> np.ndarray:
+        """Encode texts to embeddings."""
+        pass
+    
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Get display name for the model."""
+        pass
+    
+    @property
+    @abstractmethod
+    def description(self) -> str:
+        """Get description for the model."""
+        pass
+    
+    def encode_pairs(
+        self,
+        he_texts: list[str],
+        en_texts: list[str],
+        batch_size: int = 32,
+        show_progress: bool = True,
+    ) -> tuple[np.ndarray, np.ndarray]:
+        """
+        Encode parallel Hebrew/English text pairs.
+        
+        Args:
+            he_texts: Hebrew/Aramaic source texts
+            en_texts: English translations
+            batch_size: Batch size for encoding
+            show_progress: Whether to show progress bar
+            
+        Returns:
+            Tuple of (hebrew_embeddings, english_embeddings)
+        """
+        he_embeddings = self.encode(
+            he_texts,
+            is_query=True,
+            batch_size=batch_size,
+            show_progress=show_progress,
+        )
+        
+        en_embeddings = self.encode(
+            en_texts,
+            is_query=False,
+            batch_size=batch_size,
+            show_progress=show_progress,
+        )
+        
+        return he_embeddings, en_embeddings
+
+
+class EmbeddingModel(BaseEmbeddingModel):
+    """
+    Wrapper for sentence-transformer models with consistent interface.
+    """
+    
+    def __init__(
+        self,
+        model_id: str,
+        device: Optional[str] = None,
+        max_length: int = 512,
+    ):
+        """
+        Initialize the embedding model.
+        
+        Args:
+            model_id: Hugging Face model ID
+            device: Device to use ('cuda', 'cpu', or None for auto)
+            max_length: Maximum sequence length for tokenization
+        """
+        from sentence_transformers import SentenceTransformer
+        import torch
+        
+        self.model_id = model_id
+        self.max_length = max_length
+        
+        # Auto-detect device
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = device
+        
+        # Get model config if it's a curated model
+        self.config = CURATED_MODELS.get(model_id, {
+            "name": model_id.split("/")[-1],
+            "description": "Custom model",
+            "type": "local",
+            "query_prefix": "",
+            "passage_prefix": "",
+        })
+        
+        # Load the model
+        print(f"Loading model: {model_id} on {device}")
+        self.model = SentenceTransformer(model_id, device=device)
+        
+        # Set max sequence length if supported
+        if hasattr(self.model, "max_seq_length"):
+            self.model.max_seq_length = min(max_length, self.model.max_seq_length)
+        
+        self.embedding_dim = self.model.get_sentence_embedding_dimension()
+        print(f"Model loaded. Embedding dimension: {self.embedding_dim}")
+    
+    def encode(
+        self,
+        texts: list[str],
+        is_query: bool = False,
+        batch_size: int = 32,
+        show_progress: bool = True,
+        normalize: bool = True,
+    ) -> np.ndarray:
+        """
+        Encode texts to embeddings.
+        
+        Args:
+            texts: List of texts to encode
+            is_query: Whether these are queries (vs passages) for asymmetric models
+            batch_size: Batch size for encoding
+            show_progress: Whether to show progress bar
+            normalize: Whether to L2-normalize embeddings
+            
+        Returns:
+            numpy array of shape (len(texts), embedding_dim)
+        """
+        # Add prefix if needed (for E5-style models)
+        prefix = self.config["query_prefix"] if is_query else self.config["passage_prefix"]
+        if prefix:
+            texts = [prefix + t for t in texts]
+        
+        embeddings = self.model.encode(
+            texts,
+            batch_size=batch_size,
+            show_progress_bar=show_progress,
+            normalize_embeddings=normalize,
+            convert_to_numpy=True,
+        )
+        
+        return embeddings
+    
+    @property
+    def name(self) -> str:
+        """Get display name for the model."""
+        return self.config.get("name", self.model_id)
+    
+    @property
+    def description(self) -> str:
+        """Get description for the model."""
+        return self.config.get("description", "")
+
+
+class OpenAIEmbeddingModel(BaseEmbeddingModel):
+    """
+    Wrapper for OpenAI embedding API with consistent interface.
+    """
+    
+    # OpenAI embedding models have an 8191 token limit
+    MAX_TOKENS = 8191
+    
+    def __init__(
+        self,
+        model_id: str,
+        api_key: Optional[str] = None,
+    ):
+        """
+        Initialize the OpenAI embedding model.
+        
+        Args:
+            model_id: Model ID in format 'openai/model-name'
+            api_key: OpenAI API key (or uses OPENAI_API_KEY env var)
+        """
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError(
+                "OpenAI package not installed. Install with: pip install openai"
+            )
+        
+        self.model_id = model_id
+        
+        # Get API key from parameter or environment
+        api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "OpenAI API key required. Set OPENAI_API_KEY environment variable "
+                "or pass api_key parameter."
+            )
+        
+        self.client = OpenAI(api_key=api_key)
+        
+        # Get model config
+        self.config = API_MODELS.get(model_id, {
+            "name": model_id,
+            "description": "OpenAI embedding model",
+            "type": "openai",
+            "model_name": model_id.replace("openai/", ""),
+            "dimensions": 1536,
+        })
+        
+        self._model_name = self.config["model_name"]
+        self.embedding_dim = self.config["dimensions"]
+        
+        # Initialize tokenizer for truncation
+        self._encoding = None
+        try:
+            import tiktoken
+            self._encoding = tiktoken.encoding_for_model(self._model_name)
+        except Exception:
+            # Fall back to cl100k_base which is used by embedding models
+            try:
+                import tiktoken
+                self._encoding = tiktoken.get_encoding("cl100k_base")
+            except Exception:
+                print("Warning: tiktoken not available, using character-based truncation")
+        
+        print(f"Initialized OpenAI embedding model: {self._model_name}")
+        print(f"Embedding dimension: {self.embedding_dim}")
+    
+    def _truncate_text(self, text: str) -> str:
+        """Truncate text to fit within token limit."""
+        if self._encoding is not None:
+            # Use tiktoken for accurate token counting
+            tokens = self._encoding.encode(text)
+            if len(tokens) > self.MAX_TOKENS:
+                tokens = tokens[:self.MAX_TOKENS]
+                return self._encoding.decode(tokens)
+            return text
+        else:
+            # Fallback: rough character-based truncation
+            # Assume ~3 chars per token for Hebrew/mixed text (conservative)
+            max_chars = self.MAX_TOKENS * 3
+            if len(text) > max_chars:
+                return text[:max_chars]
+            return text
+    
+    def encode(
+        self,
+        texts: list[str],
+        is_query: bool = False,
+        batch_size: int = 100,  # OpenAI supports larger batches
+        show_progress: bool = True,
+        normalize: bool = True,
+    ) -> np.ndarray:
+        """
+        Encode texts to embeddings using OpenAI API.
+        
+        Args:
+            texts: List of texts to encode
+            is_query: Not used for OpenAI (symmetric embeddings)
+            batch_size: Batch size for API calls
+            show_progress: Whether to show progress bar
+            normalize: Whether to L2-normalize embeddings (OpenAI already normalizes)
+            
+        Returns:
+            numpy array of shape (len(texts), embedding_dim)
+        """
+        import time
+        
+        all_embeddings = []
+        total_batches = (len(texts) + batch_size - 1) // batch_size
+        
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_num = i // batch_size + 1
+            
+            if show_progress:
+                print(f"  Encoding batch {batch_num}/{total_batches}...")
+            
+            # Retry logic for API calls
+            max_retries = 3
+            for attempt in range(max_retries):
+                try:
+                    response = self.client.embeddings.create(
+                        model=self._model_name,
+                        input=batch,
+                    )
+                    
+                    # Extract embeddings from response
+                    batch_embeddings = [item.embedding for item in response.data]
+                    all_embeddings.extend(batch_embeddings)
+                    break
+                    
+                except Exception as e:
+                    if attempt < max_retries - 1:
+                        wait_time = 2 ** attempt
+                        print(f"  API error, retrying in {wait_time}s: {e}")
+                        time.sleep(wait_time)
+                    else:
+                        raise RuntimeError(f"OpenAI API error after {max_retries} retries: {e}")
+            
+            # Small delay to avoid rate limits
+            if i + batch_size < len(texts):
+                time.sleep(0.1)
+        
+        embeddings = np.array(all_embeddings, dtype=np.float32)
+        
+        # OpenAI embeddings are already normalized, but normalize if requested
+        if normalize:
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            embeddings = embeddings / np.maximum(norms, 1e-10)
+        
+        return embeddings
+    
+    @property
+    def name(self) -> str:
+        """Get display name for the model."""
+        return self.config.get("name", self.model_id)
+    
+    @property
+    def description(self) -> str:
+        """Get description for the model."""
+        return self.config.get("description", "")
+
+
+class VoyageEmbeddingModel(BaseEmbeddingModel):
+    """
+    Wrapper for Voyage AI embedding API with consistent interface.
+    """
+    
+    def __init__(
+        self,
+        model_id: str,
+        api_key: Optional[str] = None,
+    ):
+        """
+        Initialize the Voyage AI embedding model.
+        
+        Args:
+            model_id: Model ID in format 'voyage/model-name'
+            api_key: Voyage API key (or uses VOYAGE_API_KEY env var)
+        """
+        try:
+            import voyageai
+        except ImportError:
+            raise ImportError(
+                "Voyage AI package not installed. Install with: pip install voyageai"
+            )
+        
+        self.model_id = model_id
+        
+        # Get API key from parameter or environment
+        api_key = api_key or os.environ.get("VOYAGE_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "Voyage API key required. Set VOYAGE_API_KEY environment variable "
+                "or pass api_key parameter."
+            )
+        
+        self.client = voyageai.Client(api_key=api_key)
+        
+        # Get model config
+        self.config = API_MODELS.get(model_id, {
+            "name": model_id,
+            "description": "Voyage AI embedding model",
+            "type": "voyage",
+            "model_name": model_id.replace("voyage/", ""),
+            "dimensions": 1024,  # Default dimension
+        })
+        
+        self._model_name = self.config["model_name"]
+        self.embedding_dim = self.config["dimensions"]
+        
+        print(f"Initialized Voyage AI embedding model: {self._model_name}")
+        print(f"Embedding dimension: {self.embedding_dim}")
+    
+    def encode(
+        self,
+        texts: list[str],
+        is_query: bool = False,
+        batch_size: int = 128,  # Voyage supports larger batches
+        show_progress: bool = True,
+        normalize: bool = True,
+    ) -> np.ndarray:
+        """
+        Encode texts to embeddings using Voyage AI API.
+        
+        Args:
+            texts: List of texts to encode
+            is_query: Whether these are queries (Voyage supports input_type)
+            batch_size: Batch size for API calls
+            show_progress: Whether to show progress bar
+            normalize: Whether to L2-normalize embeddings
+            
+        Returns:
+            numpy array of shape (len(texts), embedding_dim)
+        """
+        import time
+        
+        all_embeddings = []
+        total_batches = (len(texts) + batch_size - 1) // batch_size
+        
+        # Voyage supports input_type for asymmetric embeddings
+        input_type = "query" if is_query else "document"
+        
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_num = i // batch_size + 1
+            
+            if show_progress:
+                print(f"  Encoding batch {batch_num}/{total_batches}...")
+            
+            # Retry logic for API calls
+            max_retries = 3
+            for attempt in range(max_retries):
+                try:
+                    result = self.client.embed(
+                        batch,
+                        model=self._model_name,
+                        input_type=input_type,
+                    )
+                    
+                    # Extract embeddings from response
+                    batch_embeddings = result.embeddings
+                    all_embeddings.extend(batch_embeddings)
+                    break
+                    
+                except Exception as e:
+                    if attempt < max_retries - 1:
+                        wait_time = 2 ** attempt
+                        print(f"  API error, retrying in {wait_time}s: {e}")
+                        time.sleep(wait_time)
+                    else:
+                        raise RuntimeError(f"Voyage AI API error after {max_retries} retries: {e}")
+            
+            # Small delay to avoid rate limits
+            if i + batch_size < len(texts):
+                time.sleep(0.1)
+        
+        embeddings = np.array(all_embeddings, dtype=np.float32)
+        
+        # Normalize if requested
+        if normalize:
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            embeddings = embeddings / np.maximum(norms, 1e-10)
+        
+        return embeddings
+    
+    @property
+    def name(self) -> str:
+        """Get display name for the model."""
+        return self.config.get("name", self.model_id)
+    
+    @property
+    def description(self) -> str:
+        """Get description for the model."""
+        return self.config.get("description", "")
+
+
+class GeminiEmbeddingModel(BaseEmbeddingModel):
+    """
+    Wrapper for Google Gemini embedding API with consistent interface.
+    """
+    
+    def __init__(
+        self,
+        model_id: str,
+        api_key: Optional[str] = None,
+    ):
+        """
+        Initialize the Gemini embedding model.
+        
+        Args:
+            model_id: Model ID in format 'gemini/model-name'
+            api_key: Gemini API key (optional - can use GEMINI_API_KEY env var 
+                     or Google Cloud Application Default Credentials)
+        """
+        try:
+            from google import genai
+        except ImportError:
+            raise ImportError(
+                "Google GenAI package not installed. Install with: pip install google-genai"
+            )
+        
+        self.model_id = model_id
+        
+        # Get API key from parameter or environment (optional - ADC also works)
+        api_key = api_key or os.environ.get("GEMINI_API_KEY")
+        
+        # Create client - if no API key, will use Application Default Credentials
+        if api_key:
+            self.client = genai.Client(api_key=api_key)
+        else:
+            # Use Application Default Credentials (gcloud auth application-default login)
+            self.client = genai.Client()
+        
+        # Get model config
+        self.config = API_MODELS.get(model_id, {
+            "name": model_id,
+            "description": "Gemini embedding model",
+            "type": "gemini",
+            "model_name": model_id.replace("gemini/", "").split("-768")[0].split("-1536")[0],
+            "dimensions": 3072,  # Default dimension
+        })
+        
+        self._model_name = self.config["model_name"]
+        self.embedding_dim = self.config["dimensions"]
+        
+        print(f"Initialized Gemini embedding model: {self._model_name}")
+        print(f"Embedding dimension: {self.embedding_dim}")
+    
+    def encode(
+        self,
+        texts: list[str],
+        is_query: bool = False,
+        batch_size: int = 20,  # Smaller batches to avoid rate limits
+        show_progress: bool = True,
+        normalize: bool = True,
+    ) -> np.ndarray:
+        """
+        Encode texts to embeddings using Gemini API.
+        
+        Args:
+            texts: List of texts to encode
+            is_query: Whether these are queries (uses RETRIEVAL_QUERY vs RETRIEVAL_DOCUMENT)
+            batch_size: Batch size for API calls (smaller for Gemini to avoid rate limits)
+            show_progress: Whether to show progress bar
+            normalize: Whether to L2-normalize embeddings
+            
+        Returns:
+            numpy array of shape (len(texts), embedding_dim)
+        """
+        import time
+        import random
+        from google.genai import types
+        
+        all_embeddings = []
+        total_batches = (len(texts) + batch_size - 1) // batch_size
+        
+        # Gemini supports task_type for asymmetric embeddings
+        task_type = "RETRIEVAL_QUERY" if is_query else "RETRIEVAL_DOCUMENT"
+        
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_num = i // batch_size + 1
+            
+            if show_progress:
+                print(f"  Encoding batch {batch_num}/{total_batches}...")
+            
+            # Retry logic with exponential backoff for rate limits
+            max_retries = 8
+            base_delay = 2.0
+            
+            for attempt in range(max_retries):
+                try:
+                    # Build config with task type and output dimensionality
+                    embed_config = types.EmbedContentConfig(
+                        task_type=task_type,
+                        output_dimensionality=self.embedding_dim,
+                    )
+                    
+                    result = self.client.models.embed_content(
+                        model=self._model_name,
+                        contents=batch,
+                        config=embed_config,
+                    )
+                    
+                    # Extract embeddings from response
+                    batch_embeddings = [e.values for e in result.embeddings]
+                    all_embeddings.extend(batch_embeddings)
+                    break
+                    
+                except Exception as e:
+                    error_str = str(e)
+                    is_rate_limit = "429" in error_str or "RESOURCE_EXHAUSTED" in error_str
+                    
+                    if attempt < max_retries - 1:
+                        # Exponential backoff with jitter
+                        # Longer waits for rate limit errors
+                        if is_rate_limit:
+                            wait_time = base_delay * (2 ** attempt) + random.uniform(1, 5)
+                            print(f"  Rate limited, waiting {wait_time:.1f}s before retry {attempt + 2}/{max_retries}...")
+                        else:
+                            wait_time = base_delay * (2 ** attempt) + random.uniform(0, 1)
+                            print(f"  API error, retrying in {wait_time:.1f}s: {e}")
+                        time.sleep(wait_time)
+                    else:
+                        raise RuntimeError(f"Gemini API error after {max_retries} retries: {e}")
+            
+            # Delay between batches to avoid rate limits (longer for Gemini)
+            if i + batch_size < len(texts):
+                time.sleep(0.5)
+        
+        embeddings = np.array(all_embeddings, dtype=np.float32)
+        
+        # Normalize if requested (Gemini's 3072d is normalized, but smaller dims need it)
+        if normalize:
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            embeddings = embeddings / np.maximum(norms, 1e-10)
+        
+        return embeddings
+    
+    @property
+    def name(self) -> str:
+        """Get display name for the model."""
+        return self.config.get("name", self.model_id)
+    
+    @property
+    def description(self) -> str:
+        """Get description for the model."""
+        return self.config.get("description", "")
+
+
+def get_curated_model_choices() -> list[tuple[str, str]]:
+    """
+    Get list of curated local models for UI dropdown.
+    
+    Returns:
+        List of (model_id, display_name) tuples
+    """
+    return [
+        (model_id, f"{info['name']} - {info['description']}")
+        for model_id, info in CURATED_MODELS.items()
+    ]
+
+
+def get_api_model_choices() -> list[tuple[str, str]]:
+    """
+    Get list of API-based models for UI dropdown.
+    
+    Returns:
+        List of (model_id, display_name) tuples
+    """
+    return [
+        (model_id, f"{info['name']} - {info['description']}")
+        for model_id, info in API_MODELS.items()
+    ]
+
+
+def get_all_model_choices() -> list[tuple[str, str]]:
+    """
+    Get list of all models (local + API) for UI dropdown.
+    
+    Returns:
+        List of (model_id, display_name) tuples
+    """
+    return get_curated_model_choices() + get_api_model_choices()
+
+
+def is_api_model(model_id: str) -> bool:
+    """Check if a model ID is an API-based model."""
+    model_id = model_id.strip()
+    
+    # Check if it's in API_MODELS
+    if model_id in API_MODELS:
+        return True
+    
+    # Check if it starts with known API prefixes
+    if model_id.startswith("openai/"):
+        return True
+    if model_id.startswith("voyage/"):
+        return True
+    if model_id.startswith("gemini/"):
+        return True
+    
+    return False
+
+
+def load_model(
+    model_id: str,
+    device: Optional[str] = None,
+    api_key: Optional[str] = None,
+) -> BaseEmbeddingModel:
+    """
+    Load an embedding model by ID.
+    
+    Args:
+        model_id: Model ID (HuggingFace model ID or API model like 'openai/text-embedding-3-large')
+        device: Device to use (for local models only)
+        api_key: API key (for API-based models, or uses environment variable)
+        
+    Returns:
+        Loaded embedding model instance
+    """
+    model_id = model_id.strip()
+    
+    # Check if this is an API model
+    if is_api_model(model_id):
+        # Check model type from config or prefix
+        model_config = API_MODELS.get(model_id, {})
+        model_type = model_config.get("type", "")
+        
+        if model_type == "voyage" or model_id.startswith("voyage/"):
+            return VoyageEmbeddingModel(model_id, api_key=api_key)
+        elif model_type == "gemini" or model_id.startswith("gemini/"):
+            return GeminiEmbeddingModel(model_id, api_key=api_key)
+        elif model_type == "openai" or model_id.startswith("openai/"):
+            return OpenAIEmbeddingModel(model_id, api_key=api_key)
+        else:
+            raise ValueError(f"Unknown API model type: {model_id}")
+    
+    # Otherwise, load as a local sentence-transformer model
+    return EmbeddingModel(model_id, device=device)
+
+
+def validate_model_id(model_id: str) -> tuple[bool, str]:
+    """
+    Check if a model ID is valid and loadable.
+    
+    Args:
+        model_id: The model ID to validate
+        
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    if not model_id or not model_id.strip():
+        return False, "Model ID cannot be empty"
+    
+    model_id = model_id.strip()
+    
+    # Check if it's a curated local model
+    if model_id in CURATED_MODELS:
+        return True, ""
+    
+    # Check if it's a known API model
+    if model_id in API_MODELS:
+        return True, ""
+    
+    # Check for OpenAI models
+    if model_id.startswith("openai/"):
+        return True, ""
+    
+    # Check for Voyage AI models
+    if model_id.startswith("voyage/"):
+        return True, ""
+    
+    # Check for Gemini models
+    if model_id.startswith("gemini/"):
+        return True, ""
+    
+    # For custom models, check if it looks like a valid HF model ID
+    if "/" not in model_id:
+        return False, "Model ID should be in format 'organization/model-name'"
+    
+    # Could add an API check here, but that would slow down validation
+    return True, ""
+
+
+def requires_api_key(model_id: str) -> bool:
+    """Check if a model requires an API key."""
+    return is_api_model(model_id)
+
+
+def api_key_optional(model_id: str) -> bool:
+    """
+    Check if an API key is optional for this model.
+    
+    Some providers (like Google Gemini) support Application Default Credentials
+    as an alternative to explicit API keys.
+    """
+    key_type = get_api_key_type(model_id)
+    # Gemini supports ADC (gcloud auth application-default login)
+    return key_type == "gemini"
+
+
+def get_api_key_type(model_id: str) -> Optional[str]:
+    """
+    Get the type of API key required for a model.
+    
+    Args:
+        model_id: The model ID
+        
+    Returns:
+        'openai', 'voyage', or None if no API key needed
+    """
+    if not is_api_model(model_id):
+        return None
+    
+    model_id = model_id.strip()
+    model_config = API_MODELS.get(model_id, {})
+    model_type = model_config.get("type", "")
+    
+    if model_type == "voyage" or model_id.startswith("voyage/"):
+        return "voyage"
+    elif model_type == "gemini" or model_id.startswith("gemini/"):
+        return "gemini"
+    elif model_type == "openai" or model_id.startswith("openai/"):
+        return "openai"
+    
+    return None
+
+
+def get_api_key_env_var(model_id: str) -> Optional[str]:
+    """
+    Get the environment variable name for the API key required by a model.
+    
+    Args:
+        model_id: The model ID
+        
+    Returns:
+        Environment variable name or None
+    """
+    key_type = get_api_key_type(model_id)
+    if key_type == "openai":
+        return "OPENAI_API_KEY"
+    elif key_type == "voyage":
+        return "VOYAGE_API_KEY"
+    elif key_type == "gemini":
+        return "GEMINI_API_KEY"
+    return None
+
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(
+        description="Test embedding model loading and encoding"
+    )
+    parser.add_argument(
+        "--local",
+        action="store_true",
+        help="Test only local sentence-transformer models",
+    )
+    parser.add_argument(
+        "--remote",
+        action="store_true",
+        help="Test only remote/API models (requires API keys)",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Test a specific model ID",
+    )
+    
+    args = parser.parse_args()
+    
+    # If neither flag specified, test both
+    test_local = args.local or (not args.local and not args.remote)
+    test_remote = args.remote or (not args.local and not args.remote)
+    
+    print("Testing model loading...")
+    
+    print(f"\nLocal models available:")
+    for model_id, display in get_curated_model_choices():
+        print(f"  - {display}")
+    
+    print(f"\nAPI models available:")
+    for model_id, display in get_api_model_choices():
+        print(f"  - {display}")
+    
+    # Test texts
+    test_texts = [
+        "בראשית ברא אלהים את השמים ואת הארץ",
+        "In the beginning God created the heaven and the earth",
+    ]
+    
+    def run_model_test(model_id: str, model_type: str):
+        """Run a test for a specific model."""
+        print(f"\n{'='*60}")
+        print(f"Testing {model_type}: {model_id}")
+        print("="*60)
+        
+        try:
+            model = load_model(model_id)
+            
+            embeddings = model.encode(test_texts, show_progress=False)
+            print(f"\nEncoded {len(test_texts)} texts")
+            print(f"Embedding shape: {embeddings.shape}")
+            
+            similarity = np.dot(embeddings[0], embeddings[1])
+            print(f"Cosine similarity between Hebrew and English: {similarity:.4f}")
+            return True
+        except Exception as e:
+            print(f"Test failed: {e}")
+            return False
+    
+    # Test specific model if provided
+    if args.model:
+        run_model_test(args.model, "specified model")
+    else:
+        # Test local model
+        if test_local:
+            run_model_test(
+                "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+                "local sentence-transformer model"
+            )
+        
+        # Test API models
+        if test_remote:
+            # Test OpenAI model
+            if os.environ.get("OPENAI_API_KEY"):
+                run_model_test(
+                    "openai/text-embedding-3-small",
+                    "OpenAI API model"
+                )
+            else:
+                print("\n(Skipping OpenAI test - OPENAI_API_KEY not set)")
+            
+            # Test Voyage AI model
+            if os.environ.get("VOYAGE_API_KEY"):
+                run_model_test(
+                    "voyage/voyage-3.5",
+                    "Voyage AI API model"
+                )
+            else:
+                print("\n(Skipping Voyage AI test - VOYAGE_API_KEY not set)")
+            
+            # Test Gemini model
+            if os.environ.get("GEMINI_API_KEY"):
+                run_model_test(
+                    "gemini/gemini-embedding-001",
+                    "Gemini API model"
+                )
+            else:
+                print("\n(Skipping Gemini test - GEMINI_API_KEY not set)")
+

remove_oversize_entries.py

@@ -0,0 +1,51 @@
+"""
+One-time script to remove entries exceeding the OpenAI embedding token limit
+from the benchmark dataset.
+"""
+
+import json
+
+# The refs to remove (from the token limit check report)
+REFS_TO_REMOVE = [
+    "Shemot Rabbah.1:1",
+    "Bamidbar Rabbah.1:2",
+    "Bamidbar Rabbah.2:10",
+    "Shir HaShirim Rabbah.1.1:10",
+    "Eichah Rabbah.1:4",
+    "Eichah Rabbah.1:23",
+    "Eichah Rabbah.1:31",
+    "Ramban on Genesis.18:1",
+    "Ramban on Genesis.24:2",
+    "Ramban on Leviticus.1.9:1",
+    "Ramban on Numbers.16:1",
+    "Ramban on Numbers.24:1",
+    "Ramban on Deuteronomy.2.23:1",
+]
+
+def main():
+    data_path = "benchmark_data/benchmark.json"
+    
+    # Load the data
+    print(f"Loading data from: {data_path}")
+    with open(data_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    
+    original_count = len(data)
+    print(f"Original entry count: {original_count}")
+    
+    # Filter out the flagged entries
+    filtered_data = [entry for entry in data if entry["ref"] not in REFS_TO_REMOVE]
+    
+    removed_count = original_count - len(filtered_data)
+    print(f"Removed {removed_count} entries")
+    print(f"New entry count: {len(filtered_data)}")
+    
+    # Save the filtered data
+    print(f"Saving filtered data to: {data_path}")
+    with open(data_path, "w", encoding="utf-8") as f:
+        json.dump(filtered_data, f, ensure_ascii=False, indent=2)
+    
+    print("Done!")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,20 @@
+# Hugging Face Space dependencies
+gradio>=4.0.0
+transformers>=4.36.0
+sentence-transformers>=2.2.2
+torch>=2.0.0
+
+# Data processing
+numpy>=1.24.0
+pandas>=2.0.0
+requests>=2.31.0
+
+# Visualization
+plotly>=5.18.0
+
+# API-based embedding providers
+openai>=1.0.0
+tiktoken>=0.5.0
+voyageai>=0.3.0
+google-genai>=1.0.0
+

space_README.md

@@ -0,0 +1,48 @@
+---
+title: Rabbinic Embedding Benchmark
+emoji: 📚
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+# Rabbinic Hebrew/Aramaic Embedding Benchmark
+
+Evaluate embedding models on cross-lingual retrieval between Hebrew/Aramaic source texts and their English translations from Sefaria.
+
+## How It Works
+
+Given a Hebrew/Aramaic text, can the model find its correct English translation from a pool of candidates? Models that excel at this task produce high-quality embeddings for Rabbinic literature.
+
+## Metrics
+
+| Metric | Description |
+|--------|-------------|
+| **MRR** | Mean Reciprocal Rank (average of 1/rank of correct answer) |
+| **Recall@k** | % of queries where correct translation is in top k results |
+| **Bitext Accuracy** | True pair vs random pair classification |
+
+## Corpus
+
+The benchmark includes diverse texts with English translations:
+
+- **Talmud**: Bavli & Yerushalmi
+- **Mishnah**: Selected tractates
+- **Midrash**: Midrash Rabbah
+- **Commentary**: Rashi, Ramban, Radak, Rabbeinu Behaye
+- **Philosophy**: Guide for the Perplexed, Sefer HaIkkarim
+- **Hasidic/Kabbalistic**: Likutei Moharan, Tomer Devorah, Kalach Pitchei Chokhmah
+- **Mussar**: Chafetz Chaim, Kav HaYashar, Iggeret HaRamban
+- **Halacha**: Sefer HaChinukh, Mishneh Torah
+
+All texts sourced from [Sefaria](https://www.sefaria.org).
+
+## API Keys
+
+For API-based models (OpenAI, Voyage AI, Gemini), you can either:
+- Enter your API key in the interface (not stored)
+- Set environment variables in Space settings: `OPENAI_API_KEY`, `VOYAGE_API_KEY`, `GEMINI_API_KEY`