Spaces:
Running
Running
| { | |
| "schema_version": 2, | |
| "protocol": "bench50-v0.3.1", | |
| "status": "selected_source_registry_not_measurements_or_execution_approval", | |
| "scope": "50 distinct upstream benchmark identities; variants, splits, translations, rounds and suite members are not extra benchmarks", | |
| "counts": { | |
| "rows": 50, | |
| "distinct_benchmarks": 50, | |
| "EN": 38, | |
| "PL": 12, | |
| "source_members": 204, | |
| "dataset_repositories": 44 | |
| }, | |
| "distinctness": { | |
| "key": "benchmark_id and reviewed upstream collection, not YAML task name", | |
| "umbrella_suites_not_added_to_components": true, | |
| "statistical_independence_claimed": false, | |
| "language_balance": "source diversity first; original user directive did not impose25PL/25EN" | |
| }, | |
| "sources": { | |
| "lm_eval_commit": "6d2abda4fd171e68a8789330c4149e37c1ca0bda", | |
| "openpl_commit": "21d0ea9cf4fd6153dfff4d84d6ad0aab5488f302", | |
| "source_registry": "benchmark_sources_v03.json", | |
| "origins": { | |
| "en": "canonical pinned EN checkout", | |
| "pl": "canonical pinned PL checkout", | |
| "registry": "this release tasks/ definitions" | |
| } | |
| }, | |
| "selection": { | |
| "seed": 20261028, | |
| "max_documents_per_benchmark": 200, | |
| "unit": "source document after the original process_docs; group member namespace included", | |
| "ordering": "SHA256(UTF8(20261028|benchmark_id|member_task|source_id)); ties by canonical source bytes", | |
| "when": "seal all requested benchmark/member data, source IDs, support and requests before loading model weights", | |
| "group_allocation": "reserve min(2,eligible_population) per member; then allocate each remaining slot to the non-full member with largest (budget*source_weight - allocated), ties by member task name; never200 per member", | |
| "group_weighting": "source macro/equal-member or eligible-population-size weights retained independently of sample quota; reaggregate member metrics with those fixed weights", | |
| "support_policy": "configured source count; hash-selected disjoint support per member unless source supplies fixed examples, which retain source order; record every demo and exclude matching scored inputs", | |
| "context_policy": "float32,batch1,max4096,no automaticBOS/chat; reserve task generation limit (default256); drop optional demos from end, never query/options; explicit failure if still over limit", | |
| "wikitext": "full source test stream, canonical weighted byte_perplexity, not Glint conversion; descriptive comparison only", | |
| "full_protocol_B": "separate full-data evaluation of these selected primary protocols; not the old23-definition OpenPL replication", | |
| "harness_seed_tuple": { | |
| "python": 0, | |
| "numpy": 1234, | |
| "torch": 1234, | |
| "fewshot": 1234 | |
| }, | |
| "empty_group_member": "explicit failure; never silently remove or reweight a required source member" | |
| }, | |
| "rows": [ | |
| { | |
| "id": "T01", | |
| "benchmark_id": "blimp", | |
| "task": "blimp", | |
| "name": "BLiMP", | |
| "language": "EN", | |
| "family": "blimp", | |
| "domain": "syntax", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "group", | |
| "split": "train", | |
| "shots": 0, | |
| "rationale": "syntax", | |
| "member_count": 67, | |
| "source_record": "blimp", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "nyu-mll/blimp", | |
| "revision": "877fba0801ffb7cbd8c39c1ff314a46f053f6036", | |
| "metadata_source": "https://huggingface.co/api/datasets/nyu-mll/blimp" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/blimp/_blimp.yaml" | |
| }, | |
| { | |
| "id": "T02", | |
| "benchmark_id": "arc", | |
| "task": "arc_challenge", | |
| "name": "ARC Challenge", | |
| "language": "EN", | |
| "family": "arc", | |
| "domain": "science_reasoning", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "science reasoning", | |
| "member_count": 1, | |
| "source_record": "arc", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/ai2_arc", | |
| "revision": "210d026faf9955653af8916fad021475a3f00453", | |
| "metadata_source": "https://huggingface.co/api/datasets/allenai/ai2_arc" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "ARC Easy and Polish ARC translation are the same counted family, not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/arc/arc_challenge.yaml" | |
| }, | |
| { | |
| "id": "T03", | |
| "benchmark_id": "wikitext2", | |
| "task": "wikitext", | |
| "name": "WikiText-2", | |
| "language": "EN", | |
| "family": "wikitext2", | |
| "domain": "language_modeling", | |
| "metric": "byte_perplexity,none", | |
| "unit": "byte_ppl", | |
| "higher_is_better": false, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "language modeling", | |
| "member_count": 1, | |
| "source_record": "wikitext2", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-3.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/wikitext_document_level", | |
| "revision": "647234772b9554e208af6c826f23b99e3cac88c8", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/wikitext_document_level" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Canonical harness byte_perplexity, not the historical Glint tp^(N/B) conversion; full evaluation stream." | |
| ], | |
| "definition": "lm_eval/tasks/wikitext/wikitext.yaml" | |
| }, | |
| { | |
| "id": "T04", | |
| "benchmark_id": "multiblimp", | |
| "task": "multiblimp_pl", | |
| "name": "MultiBLiMP-PL", | |
| "language": "PL", | |
| "family": "multiblimp", | |
| "domain": "morphosyntax", | |
| "metric": "accuracy", | |
| "unit": "percent", | |
| "higher_is_better": true, | |
| "instrument": "custom_pl", | |
| "entrypoint_kind": "custom", | |
| "split": "pol/data.tsv", | |
| "shots": 0, | |
| "rationale": "morphosyntax", | |
| "member_count": 1, | |
| "source_record": "multiblimp", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "jumelet/multiblimp", | |
| "revision": "de923efa8d2483d6b13364ee68e65308e990a991", | |
| "metadata_source": "https://huggingface.co/api/datasets/jumelet/multiblimp/revision/de923efa8d2483d6b13364ee68e65308e990a991" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Independent UD/UniMorph construction, not BLiMP translation. Preserve the explicit historical space-prefix log-probability-sum instrument." | |
| ] | |
| }, | |
| { | |
| "id": "T05", | |
| "benchmark_id": "hellaswag", | |
| "task": "hellaswag", | |
| "name": "HellaSwag", | |
| "language": "EN", | |
| "family": "hellaswag", | |
| "domain": "commonsense", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "commonsense", | |
| "member_count": 1, | |
| "source_record": "hellaswag", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "Rowan/hellaswag", | |
| "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76", | |
| "metadata_source": "https://huggingface.co/api/datasets/Rowan/hellaswag/revision/218ec52e09a7e7462a5400043bb9a69a41d06b76" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "SWAG is excluded conservatively because the collections reuse ActivityNet context sources.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/hellaswag/hellaswag.yaml" | |
| }, | |
| { | |
| "id": "T06", | |
| "benchmark_id": "piqa", | |
| "task": "piqa", | |
| "name": "PIQA", | |
| "language": "EN", | |
| "family": "piqa", | |
| "domain": "physical_reasoning", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "physical reasoning", | |
| "member_count": 1, | |
| "source_record": "piqa", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "ybisk/piqa", | |
| "revision": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0", | |
| "metadata_source": "https://huggingface.co/api/datasets/ybisk/piqa/revision/2e8ac2dffd59bac8c3c6714948f4c551a0848bb0" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/piqa/piqa.yaml" | |
| }, | |
| { | |
| "id": "T07", | |
| "benchmark_id": "winogrande", | |
| "task": "winogrande", | |
| "name": "WinoGrande", | |
| "language": "EN", | |
| "family": "winogrande", | |
| "domain": "coreference", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "coreference", | |
| "member_count": 1, | |
| "source_record": "winogrande", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/winogrande", | |
| "revision": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", | |
| "metadata_source": "https://huggingface.co/api/datasets/allenai/winogrande/revision/01e74176c63542e6b0bcb004dcdea22d94fb67b5" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/winogrande/default.yaml" | |
| }, | |
| { | |
| "id": "T08", | |
| "benchmark_id": "openbookqa", | |
| "task": "openbookqa", | |
| "name": "OpenBookQA", | |
| "language": "EN", | |
| "family": "openbookqa", | |
| "domain": "science_reasoning", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "science reasoning", | |
| "member_count": 1, | |
| "source_record": "openbookqa", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/openbookqa", | |
| "revision": "388097ea7776314e93a529163e0fea805b8a6454", | |
| "metadata_source": "https://huggingface.co/api/datasets/allenai/openbookqa/revision/388097ea7776314e93a529163e0fea805b8a6454" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/openbookqa/openbookqa.yaml" | |
| }, | |
| { | |
| "id": "T09", | |
| "benchmark_id": "sciq", | |
| "task": "sciq", | |
| "name": "SciQ", | |
| "language": "EN", | |
| "family": "sciq", | |
| "domain": "science_knowledge", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "science knowledge", | |
| "member_count": 1, | |
| "source_record": "sciq", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-nc-3.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/sciq", | |
| "revision": "2c94ad3e1aafab77146f384e23536f97a4849815", | |
| "metadata_source": "https://huggingface.co/api/datasets/sciq" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "NC declaration observed: evaluation-use basis must be independently resolved; this is not training permission or run approval." | |
| ], | |
| "definition": "lm_eval/tasks/sciq/sciq.yaml" | |
| }, | |
| { | |
| "id": "T10", | |
| "benchmark_id": "boolq", | |
| "task": "boolq", | |
| "name": "BoolQ", | |
| "language": "EN", | |
| "family": "boolq", | |
| "domain": "reading", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "reading", | |
| "member_count": 1, | |
| "source_record": "boolq", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/boolq/default.yaml" | |
| }, | |
| { | |
| "id": "T11", | |
| "benchmark_id": "copa", | |
| "task": "copa", | |
| "name": "COPA", | |
| "language": "EN", | |
| "family": "copa", | |
| "domain": "causal_reasoning", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "causal reasoning", | |
| "member_count": 1, | |
| "source_record": "copa", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/copa/default.yaml" | |
| }, | |
| { | |
| "id": "T12", | |
| "benchmark_id": "cb", | |
| "task": "cb", | |
| "name": "CommitmentBank", | |
| "language": "EN", | |
| "family": "cb", | |
| "domain": "entailment", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "entailment", | |
| "member_count": 1, | |
| "source_record": "cb", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/cb/default.yaml" | |
| }, | |
| { | |
| "id": "T13", | |
| "benchmark_id": "wic", | |
| "task": "wic", | |
| "name": "WiC", | |
| "language": "EN", | |
| "family": "wic", | |
| "domain": "word_sense", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "word sense", | |
| "member_count": 1, | |
| "source_record": "wic", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/wic/default.yaml" | |
| }, | |
| { | |
| "id": "T14", | |
| "benchmark_id": "wsc273", | |
| "task": "wsc273", | |
| "name": "Winograd Schema Challenge", | |
| "language": "EN", | |
| "family": "wsc273", | |
| "domain": "coreference", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "coreference", | |
| "member_count": 1, | |
| "source_record": "wsc273", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "ErnestSDavis/winograd_wsc", | |
| "revision": "8052c534a4cecd8328c0bc879f6a5aa07404c4da", | |
| "metadata_source": "https://huggingface.co/api/datasets/winograd_wsc" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/wsc273/default.yaml" | |
| }, | |
| { | |
| "id": "T15", | |
| "benchmark_id": "rte", | |
| "task": "sglue_rte", | |
| "name": "RTE", | |
| "language": "EN", | |
| "family": "rte", | |
| "domain": "entailment", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "entailment", | |
| "member_count": 1, | |
| "source_record": "rte", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/rte/default.yaml" | |
| }, | |
| { | |
| "id": "T16", | |
| "benchmark_id": "anli", | |
| "task": "anli_r3", | |
| "name": "ANLI (R3 primary)", | |
| "language": "EN", | |
| "family": "anli", | |
| "domain": "adversarial_entailment", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test_r3", | |
| "shots": 0, | |
| "rationale": "adversarial entailment", | |
| "member_count": 1, | |
| "source_record": "anli", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-nc-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "facebook/anli", | |
| "revision": "8e4813d81f46d313dac7892e1c28076917cfcdf9", | |
| "metadata_source": "https://huggingface.co/api/datasets/anli" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "R3 is the primary protocol; R1/R2 do not increase benchmark count.", | |
| "NC declaration observed: evaluation-use basis must be independently resolved; this is not training permission or run approval." | |
| ], | |
| "definition": "lm_eval/tasks/anli/anli_r3.yaml" | |
| }, | |
| { | |
| "id": "T17", | |
| "benchmark_id": "lambada", | |
| "task": "lambada_openai", | |
| "name": "LAMBADA", | |
| "language": "EN", | |
| "family": "lambada", | |
| "domain": "contextual_prediction", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "contextual prediction", | |
| "member_count": 1, | |
| "source_record": "lambada", | |
| "license": { | |
| "declared_tags": [ | |
| "mit" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/lambada_openai", | |
| "revision": "900124bf3b8235c6daf21033af9948b3f07346c4", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/lambada_openai" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/lambada/lambada_openai.yaml" | |
| }, | |
| { | |
| "id": "T18", | |
| "benchmark_id": "logiqa", | |
| "task": "logiqa", | |
| "name": "LogiQA", | |
| "language": "EN", | |
| "family": "logiqa", | |
| "domain": "logical_reasoning", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "logical reasoning", | |
| "member_count": 1, | |
| "source_record": "logiqa", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/logiqa", | |
| "revision": "c1eee1b2fd0ddc87777e36490f7ad54d0b97d141", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/logiqa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/logiqa/logiqa.yaml" | |
| }, | |
| { | |
| "id": "T19", | |
| "benchmark_id": "mathqa", | |
| "task": "mathqa", | |
| "name": "MathQA", | |
| "language": "EN", | |
| "family": "mathqa", | |
| "domain": "mathematics_mc", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "mathematics mc", | |
| "member_count": 1, | |
| "source_record": "mathqa", | |
| "license": { | |
| "declared_tags": [ | |
| "apache-2.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/math_qa", | |
| "revision": "c4f1cc784c04c4957b50c97858f23893b633eea6", | |
| "metadata_source": "https://huggingface.co/api/datasets/math_qa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/mathqa/mathqa.yaml" | |
| }, | |
| { | |
| "id": "T20", | |
| "benchmark_id": "prost", | |
| "task": "prost", | |
| "name": "PROST", | |
| "language": "EN", | |
| "family": "prost", | |
| "domain": "physical_reasoning", | |
| "metric": "acc_norm,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "physical reasoning", | |
| "member_count": 1, | |
| "source_record": "prost", | |
| "license": { | |
| "declared_tags": [ | |
| "apache-2.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "corypaik/prost", | |
| "revision": "b3efebf08969fc19335ba894353316878b6fa493", | |
| "metadata_source": "https://huggingface.co/api/datasets/corypaik/prost" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/prost/corypaik_prost.yaml" | |
| }, | |
| { | |
| "id": "T21", | |
| "benchmark_id": "sst2", | |
| "task": "sst2", | |
| "name": "SST-2", | |
| "language": "EN", | |
| "family": "sst2", | |
| "domain": "sentiment", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "sentiment", | |
| "member_count": 1, | |
| "source_record": "sst2", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "nyu-mll/glue", | |
| "revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", | |
| "metadata_source": "https://huggingface.co/api/datasets/glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/glue/sst2/default.yaml" | |
| }, | |
| { | |
| "id": "T22", | |
| "benchmark_id": "polemo2", | |
| "task": "polemo2_in_multiple_choice", | |
| "name": "PoLEMo2 (in-domain MC)", | |
| "language": "PL", | |
| "family": "polemo2", | |
| "domain": "sentiment", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "sentiment", | |
| "member_count": 1, | |
| "source_record": "polemo2", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allegro/klej-polemo2-in", | |
| "revision": "9843af31facd613eb091deaa4141df4331d4f918", | |
| "metadata_source": "https://huggingface.co/api/datasets/allegro/klej-polemo2-in" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polemo2/polemo2_in_mc.yaml" | |
| }, | |
| { | |
| "id": "T23", | |
| "benchmark_id": "8tags", | |
| "task": "polish_8tags_multiple_choice", | |
| "name": "8tags", | |
| "language": "PL", | |
| "family": "8tags", | |
| "domain": "topic_classification", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "topic classification", | |
| "member_count": 1, | |
| "source_record": "8tags", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-nc-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "sdadas/8tags", | |
| "revision": "82ddf9c0d06ffc982aeccf2473b7ce31f2167adf", | |
| "metadata_source": "https://huggingface.co/api/datasets/sdadas/8tags" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks.", | |
| "NC declaration observed: evaluation-use basis must be independently resolved; this is not training permission or run approval." | |
| ], | |
| "definition": "lm_eval/tasks/polish_8tags/8tags_mc.yaml" | |
| }, | |
| { | |
| "id": "T24", | |
| "benchmark_id": "belebele", | |
| "task": "polish_belebele_mc", | |
| "name": "Belebele-PL", | |
| "language": "PL", | |
| "family": "belebele", | |
| "domain": "reading", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "reading", | |
| "member_count": 1, | |
| "source_record": "belebele", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "facebook/belebele", | |
| "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", | |
| "metadata_source": "https://huggingface.co/api/datasets/facebook/belebele/revision/7899cdfa4e1e0d733fd77c848e2c273cb1d32be2" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_belebele/belebele_mc.yaml" | |
| }, | |
| { | |
| "id": "T25", | |
| "benchmark_id": "dyk", | |
| "task": "polish_dyk_multiple_choice", | |
| "name": "CzyWiesz / DYK", | |
| "language": "PL", | |
| "family": "dyk", | |
| "domain": "question_relevance", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "question relevance", | |
| "member_count": 1, | |
| "source_record": "dyk", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-3.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allegro/klej-dyk", | |
| "revision": "dd7c3c3bb336738c853ea17c64b85d876e6714a9", | |
| "metadata_source": "https://huggingface.co/api/datasets/allegro/klej-dyk" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_dyk/dyk_mc.yaml" | |
| }, | |
| { | |
| "id": "T26", | |
| "benchmark_id": "ppc", | |
| "task": "polish_ppc_multiple_choice", | |
| "name": "Polish Paraphrase Corpus", | |
| "language": "PL", | |
| "family": "ppc", | |
| "domain": "paraphrase", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "paraphrase", | |
| "member_count": 1, | |
| "source_record": "ppc", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-nc-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "sdadas/ppc", | |
| "revision": "073012dc81efb47b0fd66b0ab48bf06ddb62f528", | |
| "metadata_source": "https://huggingface.co/api/datasets/sdadas/ppc" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks.", | |
| "NC declaration observed: evaluation-use basis must be independently resolved; this is not training permission or run approval." | |
| ], | |
| "definition": "lm_eval/tasks/polish_ppc/ppc_mc.yaml" | |
| }, | |
| { | |
| "id": "T27", | |
| "benchmark_id": "psc", | |
| "task": "polish_psc_multiple_choice", | |
| "name": "Polish Summaries Corpus (matching)", | |
| "language": "PL", | |
| "family": "psc", | |
| "domain": "summary_matching", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "summary matching", | |
| "member_count": 1, | |
| "source_record": "psc", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-3.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allegro/klej-psc", | |
| "revision": "9585582aef37e7747ef174b0ec3b74855da8bad2", | |
| "metadata_source": "https://huggingface.co/api/datasets/allegro/klej-psc" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_psc/psc_mc.yaml" | |
| }, | |
| { | |
| "id": "T28", | |
| "benchmark_id": "cbd", | |
| "task": "polish_cbd_multiple_choice", | |
| "name": "Polish Cyberbullying Dataset", | |
| "language": "PL", | |
| "family": "cbd", | |
| "domain": "toxicity_detection", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "toxicity detection", | |
| "member_count": 1, | |
| "source_record": "cbd", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "ptaszynski/PolishCyberbullyingDataset", | |
| "revision": "be3b1236598a02eff6db05ea7ac386afaabc33e8", | |
| "metadata_source": "https://huggingface.co/api/datasets/ptaszynski/PolishCyberbullyingDataset" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_cbd/cbd_mc.yaml" | |
| }, | |
| { | |
| "id": "T29", | |
| "benchmark_id": "klej_ner", | |
| "task": "polish_klej_ner_multiple_choice", | |
| "name": "KLEJ NKJP-NER (sentence-level)", | |
| "language": "PL", | |
| "family": "klej_ner", | |
| "domain": "sentence_entity_classification", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "entity recognition", | |
| "member_count": 1, | |
| "source_record": "klej_ner", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allegro/klej-nkjp-ner", | |
| "revision": "bfae4047738b985bc2ad533d339dd89c3f7de00d", | |
| "metadata_source": "https://huggingface.co/api/datasets/allegro/klej-nkjp-ner/revision/bfae4047738b985bc2ad533d339dd89c3f7de00d" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/polish_klej_ner/klej_ner_mc.yaml" | |
| }, | |
| { | |
| "id": "T30", | |
| "benchmark_id": "polqa", | |
| "task": "polish_polqa_open_book", | |
| "name": "PolQA (open book)", | |
| "language": "PL", | |
| "family": "polqa", | |
| "domain": "question_answering", | |
| "metric": "levenshtein,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "question answering", | |
| "member_count": 1, | |
| "source_record": "polqa", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "ipipan/polqa", | |
| "revision": "d78d036ef08ab3b9f4d85a2893f4d3a0c95a6f37", | |
| "metadata_source": "https://huggingface.co/api/datasets/ipipan/polqa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_polqa/polqa_open_book.yaml" | |
| }, | |
| { | |
| "id": "T31", | |
| "benchmark_id": "poquad", | |
| "task": "polish_poquad_open_book", | |
| "name": "PoQuAD", | |
| "language": "PL", | |
| "family": "poquad", | |
| "domain": "question_answering", | |
| "metric": "levenshtein,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "openpl_polish4", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 5, | |
| "rationale": "question answering", | |
| "member_count": 1, | |
| "source_record": "poquad", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "clarin-pl/poquad", | |
| "revision": "a60f228d836fbbf990d11b4590c60b3e7f2aec2d", | |
| "metadata_source": "https://huggingface.co/api/datasets/clarin-pl/poquad" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Other response-format, domain or language protocols of this source are not extra main benchmarks." | |
| ], | |
| "definition": "lm_eval/tasks/polish_poquad/poquad_open_book.yaml" | |
| }, | |
| { | |
| "id": "T32", | |
| "benchmark_id": "mmlu", | |
| "task": "mmlu", | |
| "name": "MMLU", | |
| "language": "EN", | |
| "family": "mmlu", | |
| "domain": "academic_knowledge", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "group", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "academic knowledge", | |
| "member_count": 57, | |
| "source_record": "mmlu", | |
| "license": { | |
| "declared_tags": [ | |
| "mit" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "hails/mmlu_no_train", | |
| "revision": "b2e1ec9aa795adafe68e8e983248dbd4b52a1c60", | |
| "metadata_source": "https://huggingface.co/api/datasets/hails/mmlu_no_train" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "57 subjects count as one benchmark; MMLU-Pro and translations are not additional sources." | |
| ], | |
| "definition": "lm_eval/tasks/mmlu/default/_mmlu.yaml" | |
| }, | |
| { | |
| "id": "T33", | |
| "benchmark_id": "gsm8k", | |
| "task": "gsm8k", | |
| "name": "GSM8K", | |
| "language": "EN", | |
| "family": "gsm8k", | |
| "domain": "mathematics_generation", | |
| "metric": "exact_match,strict-match", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 5, | |
| "rationale": "mathematics generation", | |
| "member_count": 1, | |
| "source_record": "gsm8k", | |
| "license": { | |
| "declared_tags": [ | |
| "mit" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "openai/gsm8k", | |
| "revision": "740312add88f781978c0658806c59bc2815b9866", | |
| "metadata_source": "https://huggingface.co/api/datasets/gsm8k" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/gsm8k/gsm8k.yaml" | |
| }, | |
| { | |
| "id": "T34", | |
| "benchmark_id": "math", | |
| "task": "hendrycks_math", | |
| "name": "MATH", | |
| "language": "EN", | |
| "family": "math", | |
| "domain": "competition_mathematics", | |
| "metric": "exact_match,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "group", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "competition mathematics", | |
| "member_count": 7, | |
| "source_record": "math", | |
| "license": { | |
| "declared_tags": [ | |
| "mit" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/hendrycks_math", | |
| "revision": "21a5633873b6a120296cce3e2df9d5550074f4a3", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/hendrycks_math" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/hendrycks_math/hendrycks_math.yaml", | |
| "scoring_notes": [ | |
| "source_extractor_floor" | |
| ] | |
| }, | |
| { | |
| "id": "T35", | |
| "benchmark_id": "truthfulqa", | |
| "task": "truthfulqa_mc2", | |
| "name": "TruthfulQA (MC2)", | |
| "language": "EN", | |
| "family": "truthfulqa", | |
| "domain": "truthfulness", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "truthfulness", | |
| "member_count": 1, | |
| "source_record": "truthfulqa", | |
| "license": { | |
| "declared_tags": [ | |
| "apache-2.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "truthfulqa/truthful_qa", | |
| "revision": "741b8276f2d1982aa3d5b832d3ee81ed3b896490", | |
| "metadata_source": "https://huggingface.co/api/datasets/truthful_qa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "MC2 is normalized probability mass on true answers, not MC1 classification accuracy; source fixed QA preamble retained." | |
| ], | |
| "definition": "lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml" | |
| }, | |
| { | |
| "id": "T36", | |
| "benchmark_id": "ifeval", | |
| "task": "ifeval", | |
| "name": "IFEval", | |
| "language": "EN", | |
| "family": "ifeval", | |
| "domain": "instruction_following", | |
| "metric": "prompt_level_strict_acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "train", | |
| "shots": 0, | |
| "rationale": "instruction following", | |
| "member_count": 1, | |
| "source_record": "ifeval", | |
| "license": { | |
| "declared_tags": [ | |
| "apache-2.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "google/IFEval", | |
| "revision": "966cd89545d6b6acfd7638bc708b98261ca58e84", | |
| "metadata_source": "https://huggingface.co/api/datasets/google/IFEval" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/ifeval/ifeval.yaml" | |
| }, | |
| { | |
| "id": "T37", | |
| "benchmark_id": "llmzszl", | |
| "task": "n05_llmzszl", | |
| "name": "LLMzSzŁ", | |
| "language": "PL", | |
| "family": "llmzszl", | |
| "domain": "native_exam_knowledge", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "n05_static_yaml", | |
| "entrypoint_kind": "local_yaml", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "native exam knowledge", | |
| "member_count": 1, | |
| "source_record": "llmzszl", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "amu-cai/llmzszl-dataset", | |
| "revision": "9c11d52a415147f1e2c44038a9d4d4155f064e1a", | |
| "metadata_source": "https://huggingface.co/api/datasets/amu-cai/llmzszl-dataset" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Dataset card has no license tag: internal evaluation and aggregate export only, per the data-use review. Known training overlap requires revision/ID-bound audit and common clean-subset decision.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "tasks/llmzszl.yaml", | |
| "contamination_status": "known overlap reported; exact current revision and ID-bound clean-cohort audit required" | |
| }, | |
| { | |
| "id": "T38", | |
| "benchmark_id": "squad2", | |
| "task": "squadv2", | |
| "name": "SQuAD 2.0", | |
| "language": "EN", | |
| "family": "squad2", | |
| "domain": "unanswerable_reading", | |
| "metric": "f1,none", | |
| "unit": "percent", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "legacy_task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "unanswerable reading", | |
| "member_count": 1, | |
| "source_record": "squad2", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "rajpurkar/squad_v2", | |
| "revision": "3ffb306f725f7d2ce8394bc1873b24868140c412", | |
| "metadata_source": "https://huggingface.co/api/datasets/squad_v2" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Do not also count QNLI, XQuAD or SQuAD completion. Preserve the legacy Task and pinned squad_v2 metric, not a fabricated ConfigurableTask replacement." | |
| ], | |
| "definition": "lm_eval/tasks/squadv2/task.py" | |
| }, | |
| { | |
| "id": "T39", | |
| "benchmark_id": "drop", | |
| "task": "drop", | |
| "name": "DROP", | |
| "language": "EN", | |
| "family": "drop", | |
| "domain": "discrete_reading", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "discrete reading", | |
| "member_count": 1, | |
| "source_record": "drop", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-4.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/drop", | |
| "revision": "619386fc7ffe6dace3cc6d3483c69b164ebf6bf3", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/drop" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/drop/default.yaml" | |
| }, | |
| { | |
| "id": "T40", | |
| "benchmark_id": "race", | |
| "task": "race", | |
| "name": "RACE (high primary)", | |
| "language": "EN", | |
| "family": "race", | |
| "domain": "reading", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "reading", | |
| "member_count": 1, | |
| "source_record": "race", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/race", | |
| "revision": "66ca2e6374282f5e35b11559e87185db911a6900", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/race" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence.", | |
| "Pinned harness high protocol includes earlier article Q/A context; not a claim of full original RACE middle+high paper parity." | |
| ], | |
| "definition": "lm_eval/tasks/race/race.yaml" | |
| }, | |
| { | |
| "id": "T41", | |
| "benchmark_id": "coqa", | |
| "task": "coqa", | |
| "name": "CoQA", | |
| "language": "EN", | |
| "family": "coqa", | |
| "domain": "conversational_reading", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "conversational reading", | |
| "member_count": 1, | |
| "source_record": "coqa", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "EleutherAI/coqa", | |
| "revision": "82e11af842af6c1396f5e9a5c7de260107c50cf1", | |
| "metadata_source": "https://huggingface.co/api/datasets/EleutherAI/coqa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/coqa/default.yaml" | |
| }, | |
| { | |
| "id": "T42", | |
| "benchmark_id": "nq_open", | |
| "task": "nq_open", | |
| "name": "Natural Questions Open", | |
| "language": "EN", | |
| "family": "nq_open", | |
| "domain": "closed_book_knowledge", | |
| "metric": "exact_match,remove_whitespace", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "closed book knowledge", | |
| "member_count": 1, | |
| "source_record": "nq_open", | |
| "license": { | |
| "declared_tags": [ | |
| "cc-by-sa-3.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "google-research-datasets/nq_open", | |
| "revision": "5dd9790a83002ad084ddeb7c420dc716852c6f28", | |
| "metadata_source": "https://huggingface.co/api/datasets/nq_open" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/nq_open/nq_open.yaml" | |
| }, | |
| { | |
| "id": "T43", | |
| "benchmark_id": "triviaqa", | |
| "task": "triviaqa", | |
| "name": "TriviaQA", | |
| "language": "EN", | |
| "family": "triviaqa", | |
| "domain": "closed_book_knowledge", | |
| "metric": "exact_match,remove_whitespace", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "closed book knowledge", | |
| "member_count": 1, | |
| "source_record": "triviaqa", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "mandarjoshi/trivia_qa", | |
| "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f", | |
| "metadata_source": "https://huggingface.co/api/datasets/mandarjoshi/trivia_qa/revision/0f7faf33a3908546c6fd5b73a660e0f8ff173c2f" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/triviaqa/default.yaml" | |
| }, | |
| { | |
| "id": "T44", | |
| "benchmark_id": "pubmedqa", | |
| "task": "pubmedqa", | |
| "name": "PubMedQA", | |
| "language": "EN", | |
| "family": "pubmedqa", | |
| "domain": "biomedical_reading", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "biomedical reading", | |
| "member_count": 1, | |
| "source_record": "pubmedqa", | |
| "license": { | |
| "declared_tags": [ | |
| "mit" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "bigbio/pubmed_qa", | |
| "revision": "15578e80ef02c2c9636d33fdd7eb6b05b987ba60", | |
| "metadata_source": "https://huggingface.co/api/datasets/bigbio/pubmed_qa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/pubmedqa/pubmedqa.yaml" | |
| }, | |
| { | |
| "id": "T45", | |
| "benchmark_id": "medmcqa", | |
| "task": "medmcqa", | |
| "name": "MedMCQA", | |
| "language": "EN", | |
| "family": "medmcqa", | |
| "domain": "medical_exam_knowledge", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "medical exam knowledge", | |
| "member_count": 1, | |
| "source_record": "medmcqa", | |
| "license": { | |
| "declared_tags": [ | |
| "apache-2.0" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "openlifescienceai/medmcqa", | |
| "revision": "91c6572c454088bf71b679ad90aa8dffcd0d5868", | |
| "metadata_source": "https://huggingface.co/api/datasets/medmcqa" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [], | |
| "definition": "lm_eval/tasks/medmcqa/medmcqa.yaml" | |
| }, | |
| { | |
| "id": "T46", | |
| "benchmark_id": "social_iqa", | |
| "task": "social_iqa", | |
| "name": "Social IQa", | |
| "language": "EN", | |
| "family": "social_iqa", | |
| "domain": "social_reasoning", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "social reasoning", | |
| "member_count": 1, | |
| "source_record": "social_iqa", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "allenai/social_i_qa", | |
| "revision": "8835ceb9141d7896d9d968634a9b21ae440e3ec5", | |
| "metadata_source": "https://huggingface.co/api/datasets/allenai/social_i_qa/revision/8835ceb9141d7896d9d968634a9b21ae440e3ec5" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/siqa/siqa.yaml" | |
| }, | |
| { | |
| "id": "T47", | |
| "benchmark_id": "bbh", | |
| "task": "bbh_fewshot", | |
| "name": "BIG-Bench Hard (direct)", | |
| "language": "EN", | |
| "family": "bbh", | |
| "domain": "formal_reasoning", | |
| "metric": "exact_match,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "group", | |
| "split": "test", | |
| "shots": 3, | |
| "rationale": "formal reasoning", | |
| "member_count": 27, | |
| "source_record": "bbh", | |
| "license": { | |
| "declared_tags": [ | |
| "unknown" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "lukaemon/bbh", | |
| "revision": "982bb89fd79532a8ac676a61fc42eb1aeec63f99", | |
| "metadata_source": "https://huggingface.co/api/datasets/lukaemon/bbh/revision/982bb89fd79532a8ac676a61fc42eb1aeec63f99" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "27 subtasks count once; direct source-provided fixed examples, not an additional CoT benchmark.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/bbh/fewshot/_bbh_fewshot.yaml", | |
| "scoring_amendments": [ | |
| "answer_leading_space_strip" | |
| ] | |
| }, | |
| { | |
| "id": "T48", | |
| "benchmark_id": "paws", | |
| "task": "paws_en", | |
| "name": "PAWS (English PAWS-X protocol)", | |
| "language": "EN", | |
| "family": "paws", | |
| "domain": "adversarial_paraphrase", | |
| "metric": "acc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "test", | |
| "shots": 0, | |
| "rationale": "adversarial paraphrase", | |
| "member_count": 1, | |
| "source_record": "paws", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "google-research-datasets/paws-x", | |
| "revision": "4cd8187c404bda33cb1f62b49b001115862acf37", | |
| "metadata_source": "https://huggingface.co/api/datasets/paws-x" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/paws-x/paws_en.yaml" | |
| }, | |
| { | |
| "id": "T49", | |
| "benchmark_id": "cola", | |
| "task": "cola", | |
| "name": "CoLA", | |
| "language": "EN", | |
| "family": "cola", | |
| "domain": "acceptability", | |
| "metric": "mcc,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "acceptability", | |
| "member_count": 1, | |
| "source_record": "cola", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "nyu-mll/glue", | |
| "revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", | |
| "metadata_source": "https://huggingface.co/api/datasets/glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "MCC may be negative; do not clamp to accuracy0..1.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/glue/cola/default.yaml" | |
| }, | |
| { | |
| "id": "T50", | |
| "benchmark_id": "record", | |
| "task": "record", | |
| "name": "ReCoRD", | |
| "language": "EN", | |
| "family": "record", | |
| "domain": "entity_reading", | |
| "metric": "f1,none", | |
| "unit": "score", | |
| "higher_is_better": true, | |
| "instrument": "lm_eval_0.4.8", | |
| "entrypoint_kind": "task", | |
| "split": "validation", | |
| "shots": 0, | |
| "rationale": "entity reading", | |
| "member_count": 1, | |
| "source_record": "record", | |
| "license": { | |
| "declared_tags": [ | |
| "other" | |
| ], | |
| "basis": "revision-bound dataset-card metadata, not legal or execution approval", | |
| "source_revisions": [ | |
| { | |
| "repo_id": "aps/super_glue", | |
| "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646", | |
| "metadata_source": "https://huggingface.co/api/datasets/super_glue" | |
| } | |
| ] | |
| }, | |
| "data_gate": "required_before_evaluation", | |
| "notes": [ | |
| "News passages may overlap broad media sources of CoQA/RACE; the curated entity-cloze questions are a distinct benchmark, not a translation or answer-format variant.", | |
| "Undeclared/umbrella data terms are a gate, not an inferred permissive licence." | |
| ], | |
| "definition": "lm_eval/tasks/super_glue/record/default.yaml" | |
| } | |
| ], | |
| "verification": { | |
| "source_identities": 50, | |
| "source_members": 204, | |
| "repository_revision_metadata": 44, | |
| "declaration_files": 218, | |
| "execution_ready": false, | |
| "model_quality_measured": false | |
| } | |
| } | |