stage2/eval_surface/manifest.json

{
  "n_distractors_per_source": 3000,
  "seed": 42,
  "sources": [
    "fiqa",
    "nq",
    "hotpotqa",
    "msmarco",
    "fever",
    "squadv2",
    "trivia"
  ],
  "per_source": {
    "fiqa": {
      "source": "fiqa",
      "n_corpus": 3257,
      "n_queries": 100,
      "n_qrels": 257,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 257,
      "seed": 42,
      "split_seed": 42
    },
    "nq": {
      "source": "nq",
      "n_corpus": 3057,
      "n_queries": 100,
      "n_qrels": 57,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 57,
      "seed": 42,
      "split_seed": 42
    },
    "hotpotqa": {
      "source": "hotpotqa",
      "n_corpus": 3200,
      "n_queries": 100,
      "n_qrels": 200,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 200,
      "seed": 42,
      "split_seed": 42
    },
    "msmarco": {
      "source": "msmarco",
      "n_corpus": 3109,
      "n_queries": 100,
      "n_qrels": 109,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 109,
      "seed": 42,
      "split_seed": 42
    },
    "fever": {
      "source": "fever",
      "n_corpus": 3128,
      "n_queries": 100,
      "n_qrels": 133,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 128,
      "seed": 42,
      "split_seed": 42
    },
    "squadv2": {
      "source": "squadv2",
      "n_corpus": 3100,
      "n_queries": 100,
      "n_qrels": 100,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 100,
      "seed": 42,
      "split_seed": 42
    },
    "trivia": {
      "source": "trivia",
      "n_corpus": 4136,
      "n_queries": 100,
      "n_qrels": 1136,
      "n_distractors_requested": 3000,
      "n_distractors_realized": 3000,
      "n_eval_positives": 1136,
      "seed": 42,
      "split_seed": 42
    }
  }
}