| [ |
| { |
| "task": "consensus4", |
| "text": "Evidence:\nA controlled experiment found that the measured signal increased when the treatment was applied.\nHypothesis:\nThe treatment increases the measured signal.", |
| "kind": "score", |
| "question": "Estimate the mean human support code for this evidence and hypothesis: no support = 0, partial support = 0.5, full support = 1. Choose the stated interval. This is coded annotator consensus.", |
| "choices": [ |
| "Mean support code in [0.00, 0.25)", |
| "Mean support code in [0.25, 0.50)", |
| "Mean support code in [0.50, 0.75)", |
| "Mean support code in [0.75, 1.00]" |
| ], |
| "candidate_ids": [ |
| "0", |
| "1", |
| "2", |
| "3" |
| ] |
| }, |
| { |
| "task": "consensus5", |
| "text": "Evidence:\nA controlled experiment found that the measured signal increased when the treatment was applied.\nHypothesis:\nThe treatment increases the measured signal.", |
| "kind": "score", |
| "question": "Estimate the mean human support code for this evidence and hypothesis: no support = 0, partial support = 0.5, full support = 1. Choose the stated interval. This is coded annotator consensus.", |
| "choices": [ |
| "Mean support code in [0.00, 0.20)", |
| "Mean support code in [0.20, 0.40)", |
| "Mean support code in [0.40, 0.60)", |
| "Mean support code in [0.60, 0.80)", |
| "Mean support code in [0.80, 1.00]" |
| ], |
| "candidate_ids": [ |
| "0", |
| "1", |
| "2", |
| "3", |
| "4" |
| ] |
| }, |
| { |
| "task": "scientific_field", |
| "text": "Title: A synthetic benchmark for sorting algorithms\nAbstract: We compare algorithmic time complexity using generated arrays and prove a bound for a new sorting method.", |
| "kind": "choice", |
| "question": "Which listed broad scientific field best matches this paper?", |
| "choices": [ |
| "Computer science", |
| "Economics", |
| "Electrical engineering and systems science", |
| "Mathematics", |
| "Physics and astronomy", |
| "Quantitative biology", |
| "Quantitative finance", |
| "Statistics" |
| ], |
| "candidate_ids": [ |
| "cs", |
| "econ", |
| "eess", |
| "math", |
| "physics", |
| "q-bio", |
| "q-fin", |
| "stat" |
| ] |
| }, |
| { |
| "task": "support_boolean", |
| "text": "Evidence:\nA controlled experiment found that the measured signal increased when the treatment was applied.\nHypothesis:\nThe treatment increases the measured signal.", |
| "kind": "noul", |
| "question": "Does the supplied evidence fully support the hypothesis? Partial support is not full support.", |
| "choices": [ |
| "The evidence does not fully support the hypothesis.", |
| "The evidence fully supports the hypothesis." |
| ], |
| "candidate_ids": [ |
| "0", |
| "1" |
| ] |
| }, |
| { |
| "task": "support_grade3", |
| "text": "Evidence:\nA controlled experiment found that the measured signal increased when the treatment was applied.\nHypothesis:\nThe treatment increases the measured signal.", |
| "kind": "score", |
| "question": "Grade how much the supplied evidence supports the hypothesis.", |
| "choices": [ |
| "None: the evidence provides no support.", |
| "Partial: the evidence supports part but not all of the hypothesis.", |
| "Full: the evidence supports the complete hypothesis." |
| ], |
| "candidate_ids": [ |
| "0", |
| "1", |
| "2" |
| ] |
| }, |
| { |
| "task": "research_area", |
| "text": "Title: A synthetic benchmark for sorting algorithms\nAbstract: We compare algorithmic time complexity using generated arrays and prove a bound for a new sorting method.", |
| "kind": "choice", |
| "question": "Which research area best describes the main scientific question or contribution? Choose the primary area, not a technique merely used. Patient health, clinical care and public health belong to Medicine; biological mechanisms, organisms and ecology belong to Biology; physical phenomena and astronomy belong to Physics; general algorithms, software, hardware and computing systems belong to Computer science. If none of these four areas describes the primary question, choose Other research area outside these four. Do not force a neighboring area merely because the paper uses its tools, studies its practitioners, or mentions it as context.", |
| "choices": [ |
| "Medicine and health", |
| "Biology", |
| "Physics and astronomy", |
| "Computer science", |
| "Other research area outside these four" |
| ], |
| "candidate_ids": [ |
| "0", |
| "1", |
| "2", |
| "3", |
| "4" |
| ] |
| }, |
| { |
| "task": "study_approach", |
| "text": "Title: A synthetic benchmark for sorting algorithms\nAbstract: We compare algorithmic time complexity using generated arrays and prove a bound for a new sorting method.", |
| "kind": "choice", |
| "question": "Classify the main completed approach, not background. Human experiments assign a treatment, task or behavioral condition to living people. Observational studies analyze observed clinical, field or astronomical data. Non-human experiments manipulate animals, cells or physical systems. Choose computational/theoretical for a main model, algorithm, simulation or proof, even when tested on real data. Reviews synthesize prior work; protocols have no completed results.", |
| "choices": [ |
| "Human experiment or intervention", |
| "Observational empirical study", |
| "Non-human controlled experiment", |
| "Computational, mathematical or theoretical study", |
| "Evidence synthesis or review", |
| "Protocol, proposal or unclear study approach" |
| ], |
| "candidate_ids": [ |
| "0", |
| "1", |
| "2", |
| "3", |
| "4", |
| "5" |
| ] |
| }, |
| { |
| "task": "scientific_field", |
| "text": "Title: A synthetic benchmark for sorting algorithms\nAbstract: We compare algorithmic time complexity using generated arrays and prove a bound for a new sorting method.", |
| "kind": "choice", |
| "question": "Which listed broad scientific field best matches this paper?", |
| "choices": [ |
| "Statistics", |
| "Quantitative finance", |
| "Quantitative biology", |
| "Physics and astronomy", |
| "Mathematics", |
| "Electrical engineering and systems science", |
| "Economics", |
| "Computer science" |
| ], |
| "candidate_ids": [ |
| "stat", |
| "q-fin", |
| "q-bio", |
| "physics", |
| "math", |
| "eess", |
| "econ", |
| "cs" |
| ] |
| }, |
| { |
| "text": "A synthetic comparison of two mathematical models.", |
| "question": "Which candidate is described?", |
| "choices": [ |
| "A probabilistic model", |
| "A deterministic model" |
| ], |
| "kind": "choice" |
| } |
| ] |
|
|
