artifacts/mteb_report.json

{
  "benchmark": "MTEB",
  "dry_run": true,
  "metric_policy": {
    "primary": "ndcg_at_10",
    "two_x_definition": "error_reduction >= 2 where error = 1 - metric"
  },
  "model_family": "vinci",
  "model_id": "Cilow/Vinci",
  "pipeline": "Lattice",
  "results": [
    {
      "backend": "dry-run",
      "benchmark": "MTEB",
      "elapsed_seconds": 0.0,
      "error_metrics": {
        "mrr_at_10": 0.39,
        "ndcg_at_10": 0.29000000000000004,
        "recall_at_10": 0.21000000000000008,
        "recall_at_5": 0.3500000000000001
      },
      "metrics": {
        "mrr_at_10": 0.61,
        "ndcg_at_10": 0.71,
        "recall_at_10": 0.7899999999999999,
        "recall_at_5": 0.6499999999999999
      },
      "model_id": "Cilow/Vinci",
      "sample_limit": 10,
      "task": "SciFact",
      "track": "embedding"
    },
    {
      "backend": "dry-run",
      "benchmark": "MTEB",
      "elapsed_seconds": 0.0,
      "error_metrics": {
        "mrr_at_10": 0.33999999999999997,
        "ndcg_at_10": 0.24,
        "recall_at_10": 0.16000000000000003,
        "recall_at_5": 0.30000000000000004
      },
      "metrics": {
        "mrr_at_10": 0.66,
        "ndcg_at_10": 0.76,
        "recall_at_10": 0.84,
        "recall_at_5": 0.7
      },
      "model_id": "Cilow/Vinci",
      "sample_limit": 10,
      "task": "FiQA2018",
      "track": "embedding"
    },
    {
      "backend": "dry-run",
      "benchmark": "MTEB",
      "elapsed_seconds": 0.0,
      "error_metrics": {
        "mrr_at_10": 0.43999999999999995,
        "ndcg_at_10": 0.33999999999999997,
        "recall_at_10": 0.26,
        "recall_at_5": 0.3999999999999999
      },
      "metrics": {
        "mrr_at_10": 0.56,
        "ndcg_at_10": 0.66,
        "recall_at_10": 0.74,
        "recall_at_5": 0.6000000000000001
      },
      "model_id": "Cilow/Vinci",
      "sample_limit": 10,
      "task": "NFCorpus",
      "track": "embedding"
    }
  ],
  "sample_limit": 10,
  "task_preset": "retrieval-smoke",
  "track": "embedding"
}