lost_in_decoding_artifacts/metadata/generation_config.json

{
  "description": "Generation settings for the query-variation TSV files under msmarco-dev/, trec-dl-2019/, and trec-dl-2020/, released alongside 'Lost in Decoding? Reproducing and Stress-Testing the Look-Ahead Prior in Generative Retrieval' (SIGIR 2026, DOI: 10.1145/3805712.3808567).",
  "splits": {
    "msmarco-dev": {"num_queries": 6980, "source": "MS MARCO Passage Ranking Dev queries"},
    "trec-dl-2019": {"num_queries": 43, "source": "TREC Deep Learning 2019 passage-ranking topics"},
    "trec-dl-2020": {"num_queries": 54, "source": "TREC Deep Learning 2020 passage-ranking topics"}
  },
  "perturbation_types": [
    {"name": "misspelling", "seeded": true, "description": "Character-level misspelling injection."},
    {"name": "reordering", "seeded": true, "description": "Word-order shuffling."},
    {"name": "synonym", "seeded": true, "description": "Adversarial synonym replacement."},
    {"name": "paraphrase", "seeded": true, "description": "Back-translation and seq2seq paraphrasing."},
    {"name": "naturality", "seeded": false, "description": "Deterministic style-normalization (stop-word removal); identical output regardless of seed."}
  ],
  "file_format": {
    "extension": "tsv",
    "columns": ["qid", "original_query", "perturbed_query", "perturbation", "seed"],
    "seeded_attacks": "one row per (qid, seed) pair, five seeds stacked in a single file",
    "naturality": "one row per qid; seed column is the literal string 'deterministic'"
  },
  "generation_code": "robustness/query_variations (penha transformations) in the Lost-in-Decoding repository: https://github.com/kidist-amde/lost-in-decoding/tree/main/robustness/query_variations",
  "provenance": "Derived evaluation artifacts created from the official MS MARCO Dev queries and official TREC Deep Learning 2019/2020 topics for the Lost in Decoding study.",
  "source_paper": "Lost in Decoding? Reproducing and Stress-Testing the Look-Ahead Prior in Generative Retrieval (DOI: 10.1145/3805712.3808567, arXiv:2604.23396)",
  "source_repository": "https://github.com/kidist-amde/lost-in-decoding"
}