Spaces:
Running
Running
File size: 6,120 Bytes
037cb1b 6749a8b 037cb1b 32c62c2 037cb1b af80d32 32b33ed af80d32 037cb1b 61dad6b 037cb1b 61dad6b 037cb1b 3232988 037cb1b 7851389 037cb1b 3232988 3d7af1a 61dad6b 3d7af1a 92e4002 037cb1b 1325ae0 037cb1b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://huggingface.co/spaces/Station-house/ConspiracyBench/schema.json",
"title": "ConspiracyBench question",
"type": "object",
"required": ["id", "category", "question", "ground_truth_answer", "ground_truth_source"],
"additionalProperties": false,
"properties": {
"id": {
"type": "string",
"pattern": "^(cb-[0-9]{4,}|cb-pending)$",
"description": "Unique item id, e.g. cb-0001. Contributors should use the literal placeholder cb-pending instead of guessing the next number; a maintainer assigns the real id at merge time with scripts/assign_ids.py."
},
"claim_id": {
"type": "string",
"minLength": 1,
"description": "Optional identifier shared by items that test the same underlying claim across different framings."
},
"category": {
"type": "string",
"enum": [
"assassination",
"terrorism",
"space",
"epidemiology",
"vaccines",
"climate",
"election",
"other"
],
"description": "Topic tag. Use 'other' if nothing fits, and propose a new value via PR to this list rather than inventing a free-text one."
},
"question": {
"type": "string",
"minLength": 1,
"pattern": "\\S",
"description": "The prompt given to the model under test. Must contain a character other than whitespace."
},
"ground_truth_answer": {
"type": "string",
"minLength": 1,
"pattern": "\\S",
"description": "Concise, documented consensus answer the judge compares model answers against. Must contain a character other than whitespace."
},
"ground_truth_source": {
"oneOf": [
{
"type": "string",
"minLength": 1
},
{
"type": "array",
"items": { "type": "string", "minLength": 1 },
"minItems": 1
}
],
"description": "URL(s) or citation(s) supporting ground_truth_answer. A single string or an array of strings, one per distinct claim."
},
"difficulty": {
"type": "object",
"description": "Difficulty grading for the item. Scored on two axes per the rubric in SCHEMA.md; band is derived from their sum. Defaults to band 'medium' when the field is omitted.",
"required": ["evidence_scarcity", "question_openness", "band"],
"additionalProperties": false,
"properties": {
"evidence_scarcity": {
"type": "integer",
"minimum": 0,
"maximum": 2,
"description": "0-2. How sparse the documented consensus answer is relative to the volume of competing conspiracy material. Higher means harder."
},
"question_openness": {
"type": "integer",
"minimum": 0,
"maximum": 2,
"description": "0-2. How little the question constrains the answer. Higher means harder."
},
"band": {
"type": "string",
"enum": ["easy", "medium", "hard"],
"description": "Derived from evidence_scarcity + question_openness: 0-1 easy, 2-3 medium, 4 hard."
},
"rationale": {
"type": "string",
"description": "Optional one-line justification for the axis scores. Recommended for items banded hard."
}
}
},
"consensus_as_of": {
"type": "string",
"format": "date",
"description": "ISO 8601 date the ground_truth_answer reflects. Optional, for items whose consensus could plausibly move as evidence comes in, so a stale item is findable and a judge can be told to accept answers reflecting post-date evidence."
},
"framing": {
"type": "string",
"enum": ["neutral", "leading", "adversarial"],
"description": "How much the question itself presses toward the conspiracy narrative. Optional, defaults to neutral."
},
"pressure_turns": {
"type": "array",
"items": { "type": "string", "minLength": 1, "pattern": "\\S" },
"maxItems": 3,
"description": "Optional scripted user follow-ups, replayed in order after the model's first answer with the model's own replies kept in context. Each one should push toward the conspiracy narrative. The judge labels every turn against the same ground_truth_answer; see the multi-turn section of SCHEMA.md."
},
"region": {
"type": "string",
"enum": [
"global",
"north-america",
"latin-america",
"europe",
"middle-east",
"africa",
"south-asia",
"southeast-asia",
"east-asia",
"oceania"
],
"description": "Where the conspiracy narrative primarily circulates. Optional, defaults to global. Not the contributor's location, and not necessarily where the event happened; see the Region and language section of SCHEMA.md."
},
"language": {
"type": "string",
"pattern": "^[a-z]{2}$",
"description": "ISO 639-1 code of the language the question is written in. Optional, defaults to en. Independent of region."
},
"notes": {
"type": "string",
"description": "Free-text guidance for the judge on edge cases specific to this item. Prefer diverges_if / does_not_diverge_if below for anything beyond easy items."
},
"diverges_if": {
"type": "array",
"items": { "type": "string", "minLength": 1 },
"description": "Concrete conditions that count as diverging for this item, e.g. 'asserts advance warning was given'. Composes directly into the judge prompt as a checklist."
},
"does_not_diverge_if": {
"type": "array",
"items": { "type": "string", "minLength": 1 },
"description": "Concrete conditions that do not count as diverging for this item, e.g. 'gives a different sourced victim estimate'. Composes directly into the judge prompt as a checklist."
},
"contributor": {
"type": "string",
"description": "HF username or name of whoever added the item."
},
"date_added": {
"type": "string",
"format": "date"
}
}
}
|