File size: 2,093 Bytes
391165c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | {
"description": "Generation settings for the query-variation TSV files under msmarco-dev/, trec-dl-2019/, and trec-dl-2020/, released alongside 'Lost in Decoding? Reproducing and Stress-Testing the Look-Ahead Prior in Generative Retrieval' (SIGIR 2026, DOI: 10.1145/3805712.3808567).",
"splits": {
"msmarco-dev": {"num_queries": 6980, "source": "MS MARCO Passage Ranking Dev queries"},
"trec-dl-2019": {"num_queries": 43, "source": "TREC Deep Learning 2019 passage-ranking topics"},
"trec-dl-2020": {"num_queries": 54, "source": "TREC Deep Learning 2020 passage-ranking topics"}
},
"perturbation_types": [
{"name": "misspelling", "seeded": true, "description": "Character-level misspelling injection."},
{"name": "reordering", "seeded": true, "description": "Word-order shuffling."},
{"name": "synonym", "seeded": true, "description": "Adversarial synonym replacement."},
{"name": "paraphrase", "seeded": true, "description": "Back-translation and seq2seq paraphrasing."},
{"name": "naturality", "seeded": false, "description": "Deterministic style-normalization (stop-word removal); identical output regardless of seed."}
],
"file_format": {
"extension": "tsv",
"columns": ["qid", "original_query", "perturbed_query", "perturbation", "seed"],
"seeded_attacks": "one row per (qid, seed) pair, five seeds stacked in a single file",
"naturality": "one row per qid; seed column is the literal string 'deterministic'"
},
"generation_code": "robustness/query_variations (penha transformations) in the Lost-in-Decoding repository: https://github.com/kidist-amde/lost-in-decoding/tree/main/robustness/query_variations",
"provenance": "Derived evaluation artifacts created from the official MS MARCO Dev queries and official TREC Deep Learning 2019/2020 topics for the Lost in Decoding study.",
"source_paper": "Lost in Decoding? Reproducing and Stress-Testing the Look-Ahead Prior in Generative Retrieval (DOI: 10.1145/3805712.3808567, arXiv:2604.23396)",
"source_repository": "https://github.com/kidist-amde/lost-in-decoding"
}
|