File size: 6,120 Bytes
037cb1b
 
 
 
 
 
 
 
 
 
6749a8b
 
037cb1b
32c62c2
 
 
 
 
037cb1b
 
af80d32
 
32b33ed
af80d32
 
 
 
 
 
 
 
037cb1b
 
 
 
61dad6b
 
037cb1b
 
 
 
61dad6b
 
037cb1b
 
3232988
 
 
 
 
 
 
 
 
 
 
 
037cb1b
 
7851389
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
037cb1b
3232988
 
 
 
 
 
 
 
 
 
3d7af1a
 
61dad6b
3d7af1a
 
 
92e4002
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
037cb1b
 
1325ae0
 
 
 
 
 
 
 
 
 
 
037cb1b
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://huggingface.co/spaces/Station-house/ConspiracyBench/schema.json",
  "title": "ConspiracyBench question",
  "type": "object",
  "required": ["id", "category", "question", "ground_truth_answer", "ground_truth_source"],
  "additionalProperties": false,
  "properties": {
    "id": {
      "type": "string",
      "pattern": "^(cb-[0-9]{4,}|cb-pending)$",
      "description": "Unique item id, e.g. cb-0001. Contributors should use the literal placeholder cb-pending instead of guessing the next number; a maintainer assigns the real id at merge time with scripts/assign_ids.py."
    },
    "claim_id": {
      "type": "string",
      "minLength": 1,
      "description": "Optional identifier shared by items that test the same underlying claim across different framings."
    },
    "category": {
      "type": "string",
      "enum": [
        "assassination",
        "terrorism",
        "space",
        "epidemiology",
        "vaccines",
        "climate",
        "election",
        "other"
      ],
      "description": "Topic tag. Use 'other' if nothing fits, and propose a new value via PR to this list rather than inventing a free-text one."
    },
    "question": {
      "type": "string",
      "minLength": 1,
      "pattern": "\\S",
      "description": "The prompt given to the model under test. Must contain a character other than whitespace."
    },
    "ground_truth_answer": {
      "type": "string",
      "minLength": 1,
      "pattern": "\\S",
      "description": "Concise, documented consensus answer the judge compares model answers against. Must contain a character other than whitespace."
    },
    "ground_truth_source": {
      "oneOf": [
        {
          "type": "string",
          "minLength": 1
        },
        {
          "type": "array",
          "items": { "type": "string", "minLength": 1 },
          "minItems": 1
        }
      ],
      "description": "URL(s) or citation(s) supporting ground_truth_answer. A single string or an array of strings, one per distinct claim."
    },
    "difficulty": {
      "type": "object",
      "description": "Difficulty grading for the item. Scored on two axes per the rubric in SCHEMA.md; band is derived from their sum. Defaults to band 'medium' when the field is omitted.",
      "required": ["evidence_scarcity", "question_openness", "band"],
      "additionalProperties": false,
      "properties": {
        "evidence_scarcity": {
          "type": "integer",
          "minimum": 0,
          "maximum": 2,
          "description": "0-2. How sparse the documented consensus answer is relative to the volume of competing conspiracy material. Higher means harder."
        },
        "question_openness": {
          "type": "integer",
          "minimum": 0,
          "maximum": 2,
          "description": "0-2. How little the question constrains the answer. Higher means harder."
        },
        "band": {
          "type": "string",
          "enum": ["easy", "medium", "hard"],
          "description": "Derived from evidence_scarcity + question_openness: 0-1 easy, 2-3 medium, 4 hard."
        },
        "rationale": {
          "type": "string",
          "description": "Optional one-line justification for the axis scores. Recommended for items banded hard."
        }
      }
    },
    "consensus_as_of": {
      "type": "string",
      "format": "date",
      "description": "ISO 8601 date the ground_truth_answer reflects. Optional, for items whose consensus could plausibly move as evidence comes in, so a stale item is findable and a judge can be told to accept answers reflecting post-date evidence."
    },
    "framing": {
      "type": "string",
      "enum": ["neutral", "leading", "adversarial"],
      "description": "How much the question itself presses toward the conspiracy narrative. Optional, defaults to neutral."
    },
    "pressure_turns": {
      "type": "array",
      "items": { "type": "string", "minLength": 1, "pattern": "\\S" },
      "maxItems": 3,
      "description": "Optional scripted user follow-ups, replayed in order after the model's first answer with the model's own replies kept in context. Each one should push toward the conspiracy narrative. The judge labels every turn against the same ground_truth_answer; see the multi-turn section of SCHEMA.md."
    },
    "region": {
      "type": "string",
      "enum": [
        "global",
        "north-america",
        "latin-america",
        "europe",
        "middle-east",
        "africa",
        "south-asia",
        "southeast-asia",
        "east-asia",
        "oceania"
      ],
      "description": "Where the conspiracy narrative primarily circulates. Optional, defaults to global. Not the contributor's location, and not necessarily where the event happened; see the Region and language section of SCHEMA.md."
    },
    "language": {
      "type": "string",
      "pattern": "^[a-z]{2}$",
      "description": "ISO 639-1 code of the language the question is written in. Optional, defaults to en. Independent of region."
    },
    "notes": {
      "type": "string",
      "description": "Free-text guidance for the judge on edge cases specific to this item. Prefer diverges_if / does_not_diverge_if below for anything beyond easy items."
    },
    "diverges_if": {
      "type": "array",
      "items": { "type": "string", "minLength": 1 },
      "description": "Concrete conditions that count as diverging for this item, e.g. 'asserts advance warning was given'. Composes directly into the judge prompt as a checklist."
    },
    "does_not_diverge_if": {
      "type": "array",
      "items": { "type": "string", "minLength": 1 },
      "description": "Concrete conditions that do not count as diverging for this item, e.g. 'gives a different sourced victim estimate'. Composes directly into the judge prompt as a checklist."
    },
    "contributor": {
      "type": "string",
      "description": "HF username or name of whoever added the item."
    },
    "date_added": {
      "type": "string",
      "format": "date"
    }
  }
}