Spaces:
Running
Running
| { | |
| "release": "bench50-v0.3.1", | |
| "kind": "contamination_field_roles", | |
| "scan": { | |
| "receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "benchmarks_section_sha256_prefix": "8be608af", | |
| "scanner_sha256": "685fa9ef1fbd6e9a680be53da7fd7d943e3c1f2cde6ef69b24271027df74dad2" | |
| }, | |
| "rule": { | |
| "context": "a supporting document shown to the model but separate from the question and the answer (passage, context, story, article, support)", | |
| "answer_bearing": "the question, the options, the gold answer or its explanation, or the scored text itself (items with no separate document: minimal pairs, sentence classification, continuation, perplexity pages)", | |
| "metadata": "annotation tags that are not shown to the model as text; hits are ignored in every variant and listed with counts", | |
| "variant_a": "items with no hit in any context or answer_bearing field", | |
| "variant_b": "items with no hit in any answer_bearing field", | |
| "extractive_qa": "answer span fields are answer_bearing even where the span is copied from the passage (SQuAD2, PoQuAD, CoQA, DROP): variant (b) is strict there and the report says so", | |
| "assigned": "from the meaning of each field in the pinned task definition, before any variant value was computed" | |
| }, | |
| "justification": { | |
| "extractive_qa": "An answer span copied from the passage is still the gold answer; one rule for every task type is preferred to a type-dependent exception, so variant (b) is stricter for extractive QA and the report says so.", | |
| "scored_text": "Where a benchmark has no separate document, the scored text is the item itself, so a hit there is a hit on the answer.", | |
| "metadata": "Morphological tags are not shown to the model as text; their hits are exact-short matches of tag strings and do not indicate exposure." | |
| }, | |
| "counts": { | |
| "context": 12, | |
| "answer_bearing": 93, | |
| "metadata": 2 | |
| }, | |
| "roles": [ | |
| { | |
| "task_id": "T44", | |
| "benchmark_id": "pubmedqa", | |
| "field": "CONTEXTS[]", | |
| "hit_items": 4, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T44", | |
| "benchmark_id": "pubmedqa", | |
| "field": "LONG_ANSWER", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T19", | |
| "benchmark_id": "mathqa", | |
| "field": "Problem", | |
| "hit_items": 74, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T44", | |
| "benchmark_id": "pubmedqa", | |
| "field": "QUESTION", | |
| "hit_items": 2, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T19", | |
| "benchmark_id": "mathqa", | |
| "field": "Rationale", | |
| "hit_items": 14, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T28", | |
| "benchmark_id": "cbd", | |
| "field": "TEXT", | |
| "hit_items": 26, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.0.input_text[]", | |
| "hit_items": 10, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.0.span_text[]", | |
| "hit_items": 39, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.1.input_text[]", | |
| "hit_items": 7, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.1.span_text[]", | |
| "hit_items": 28, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.2.input_text[]", | |
| "hit_items": 6, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "additional_answers.2.span_text[]", | |
| "hit_items": 22, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T25", | |
| "benchmark_id": "dyk", | |
| "field": "answer", | |
| "hit_items": 799, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "field": "answer.aliases[]", | |
| "hit_items": 1137, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "field": "answer.normalized_aliases[]", | |
| "hit_items": 776, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "field": "answer.normalized_value", | |
| "hit_items": 28, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "field": "answer.value", | |
| "hit_items": 62, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T46", | |
| "benchmark_id": "social_iqa", | |
| "field": "answerA", | |
| "hit_items": 2, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T46", | |
| "benchmark_id": "social_iqa", | |
| "field": "answerB", | |
| "hit_items": 4, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T42", | |
| "benchmark_id": "nq_open", | |
| "field": "answer[]", | |
| "hit_items": 64, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "answers.input_text[]", | |
| "hit_items": 4, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "answers.span_text[]", | |
| "hit_items": 36, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T31", | |
| "benchmark_id": "poquad", | |
| "field": "answers.text[]", | |
| "hit_items": 731, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T38", | |
| "benchmark_id": "squad2", | |
| "field": "answers.text[]", | |
| "hit_items": 34, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T30", | |
| "benchmark_id": "polqa", | |
| "field": "answers[]", | |
| "hit_items": 2, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T37", | |
| "benchmark_id": "llmzszl", | |
| "field": "answers[]", | |
| "hit_items": 555, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T40", | |
| "benchmark_id": "race", | |
| "field": "article", | |
| "hit_items": 75, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "attractors[].xpos", | |
| "hit_items": 3, | |
| "role": "metadata" | |
| }, | |
| { | |
| "task_id": "T11", | |
| "benchmark_id": "copa", | |
| "field": "choice1", | |
| "hit_items": 6, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T11", | |
| "benchmark_id": "copa", | |
| "field": "choice2", | |
| "hit_items": 11, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T02", | |
| "benchmark_id": "arc", | |
| "field": "choices.text[]", | |
| "hit_items": 33, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T08", | |
| "benchmark_id": "openbookqa", | |
| "field": "choices.text[]", | |
| "hit_items": 48, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T32", | |
| "benchmark_id": "mmlu", | |
| "field": "choices[]", | |
| "hit_items": 257, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "congruent_attractors[].xpos", | |
| "hit_items": 3, | |
| "role": "metadata" | |
| }, | |
| { | |
| "task_id": "T31", | |
| "benchmark_id": "poquad", | |
| "field": "context", | |
| "hit_items": 5764, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T38", | |
| "benchmark_id": "squad2", | |
| "field": "context", | |
| "hit_items": 3097, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T05", | |
| "benchmark_id": "hellaswag", | |
| "field": "ctx", | |
| "hit_items": 21, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T05", | |
| "benchmark_id": "hellaswag", | |
| "field": "ctx_a", | |
| "hit_items": 22, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T05", | |
| "benchmark_id": "hellaswag", | |
| "field": "endings[]", | |
| "hit_items": 28, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "exp", | |
| "hit_items": 33, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T27", | |
| "benchmark_id": "psc", | |
| "field": "extract_text", | |
| "hit_items": 2, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T06", | |
| "benchmark_id": "piqa", | |
| "field": "goal", | |
| "hit_items": 11, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T15", | |
| "benchmark_id": "rte", | |
| "field": "hypothesis", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T47", | |
| "benchmark_id": "bbh", | |
| "field": "input", | |
| "hit_items": 22, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T35", | |
| "benchmark_id": "truthfulqa", | |
| "field": "mc1_targets.choices[]", | |
| "hit_items": 35, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T35", | |
| "benchmark_id": "truthfulqa", | |
| "field": "mc2_targets.choices[]", | |
| "hit_items": 48, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "metadata.text", | |
| "hit_items": 164, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "opa", | |
| "hit_items": 46, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "opb", | |
| "hit_items": 41, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "opc", | |
| "hit_items": 38, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "opd", | |
| "hit_items": 85, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T19", | |
| "benchmark_id": "mathqa", | |
| "field": "options", | |
| "hit_items": 5, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T03", | |
| "benchmark_id": "wikitext2", | |
| "field": "page", | |
| "hit_items": 21, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T10", | |
| "benchmark_id": "boolq", | |
| "field": "passage", | |
| "hit_items": 637, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T39", | |
| "benchmark_id": "drop", | |
| "field": "passage", | |
| "hit_items": 2327, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T50", | |
| "benchmark_id": "record", | |
| "field": "passage", | |
| "hit_items": 189, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T30", | |
| "benchmark_id": "polqa", | |
| "field": "passage_text", | |
| "hit_items": 12667, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T30", | |
| "benchmark_id": "polqa", | |
| "field": "passage_title", | |
| "hit_items": 19, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T30", | |
| "benchmark_id": "polqa", | |
| "field": "passage_wiki", | |
| "hit_items": 13050, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "prefix", | |
| "hit_items": 41, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T11", | |
| "benchmark_id": "copa", | |
| "field": "premise", | |
| "hit_items": 9, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T15", | |
| "benchmark_id": "rte", | |
| "field": "premise", | |
| "hit_items": 9, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T16", | |
| "benchmark_id": "anli", | |
| "field": "premise", | |
| "hit_items": 19, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T34", | |
| "benchmark_id": "math", | |
| "field": "problem", | |
| "hit_items": 34, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T40", | |
| "benchmark_id": "race", | |
| "field": "problems[].question", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T36", | |
| "benchmark_id": "ifeval", | |
| "field": "prompt", | |
| "hit_items": 2, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T50", | |
| "benchmark_id": "record", | |
| "field": "query", | |
| "hit_items": 36, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T02", | |
| "benchmark_id": "arc", | |
| "field": "question", | |
| "hit_items": 15, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T09", | |
| "benchmark_id": "sciq", | |
| "field": "question", | |
| "hit_items": 15, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T10", | |
| "benchmark_id": "boolq", | |
| "field": "question", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T30", | |
| "benchmark_id": "polqa", | |
| "field": "question", | |
| "hit_items": 20, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T31", | |
| "benchmark_id": "poquad", | |
| "field": "question", | |
| "hit_items": 563, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T32", | |
| "benchmark_id": "mmlu", | |
| "field": "question", | |
| "hit_items": 405, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T35", | |
| "benchmark_id": "truthfulqa", | |
| "field": "question", | |
| "hit_items": 16, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T37", | |
| "benchmark_id": "llmzszl", | |
| "field": "question", | |
| "hit_items": 235, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T38", | |
| "benchmark_id": "squad2", | |
| "field": "question", | |
| "hit_items": 9, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T39", | |
| "benchmark_id": "drop", | |
| "field": "question", | |
| "hit_items": 4, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T42", | |
| "benchmark_id": "nq_open", | |
| "field": "question", | |
| "hit_items": 13, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "field": "question", | |
| "hit_items": 161, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "field": "question", | |
| "hit_items": 34, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T46", | |
| "benchmark_id": "social_iqa", | |
| "field": "question", | |
| "hit_items": 3, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T08", | |
| "benchmark_id": "openbookqa", | |
| "field": "question_stem", | |
| "hit_items": 26, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "questions.input_text[]", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "sen", | |
| "hit_items": 162, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T21", | |
| "benchmark_id": "sst2", | |
| "field": "sentence", | |
| "hit_items": 6, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T22", | |
| "benchmark_id": "polemo2", | |
| "field": "sentence", | |
| "hit_items": 27, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T23", | |
| "benchmark_id": "8tags", | |
| "field": "sentence", | |
| "hit_items": 243, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T29", | |
| "benchmark_id": "klej_ner", | |
| "field": "sentence", | |
| "hit_items": 138, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T49", | |
| "benchmark_id": "cola", | |
| "field": "sentence", | |
| "hit_items": 44, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T13", | |
| "benchmark_id": "wic", | |
| "field": "sentence1", | |
| "hit_items": 156, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T48", | |
| "benchmark_id": "paws", | |
| "field": "sentence1", | |
| "hit_items": 14, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T13", | |
| "benchmark_id": "wic", | |
| "field": "sentence2", | |
| "hit_items": 168, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T48", | |
| "benchmark_id": "paws", | |
| "field": "sentence2", | |
| "hit_items": 16, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T26", | |
| "benchmark_id": "ppc", | |
| "field": "sentence_A", | |
| "hit_items": 223, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T26", | |
| "benchmark_id": "ppc", | |
| "field": "sentence_B", | |
| "hit_items": 185, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T01", | |
| "benchmark_id": "blimp", | |
| "field": "sentence_bad", | |
| "hit_items": 138, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T01", | |
| "benchmark_id": "blimp", | |
| "field": "sentence_good", | |
| "hit_items": 226, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T06", | |
| "benchmark_id": "piqa", | |
| "field": "sol1", | |
| "hit_items": 48, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T06", | |
| "benchmark_id": "piqa", | |
| "field": "sol2", | |
| "hit_items": 40, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T34", | |
| "benchmark_id": "math", | |
| "field": "solution", | |
| "hit_items": 42, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T41", | |
| "benchmark_id": "coqa", | |
| "field": "story", | |
| "hit_items": 67, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T27", | |
| "benchmark_id": "psc", | |
| "field": "summary_text", | |
| "hit_items": 5, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T09", | |
| "benchmark_id": "sciq", | |
| "field": "support", | |
| "hit_items": 79, | |
| "role": "context" | |
| }, | |
| { | |
| "task_id": "T14", | |
| "benchmark_id": "wsc273", | |
| "field": "text", | |
| "hit_items": 4, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T17", | |
| "benchmark_id": "lambada", | |
| "field": "text", | |
| "hit_items": 1, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T39", | |
| "benchmark_id": "drop", | |
| "field": "validated_answers.spans[][]", | |
| "hit_items": 3, | |
| "role": "answer_bearing" | |
| }, | |
| { | |
| "task_id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "field": "wrong_sen", | |
| "hit_items": 58, | |
| "role": "answer_bearing" | |
| } | |
| ] | |
| } | |