Spaces:
Running
Running
Download rules.html from MecCog/meccog-challenge: direct link, hf CLI and curl.
- Browser
- Download file 51.5 kB
-
https://huggingface.co/spaces/MecCog/meccog-challenge/resolve/main/rules.html
- Command line
-
hf download hf://spaces/MecCog/meccog-challenge/rules.html
-
curl -L -o rules.html https://huggingface.co/spaces/MecCog/meccog-challenge/resolve/main/rules.html
51.5 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"> | |
| <title>MecCog Challenge Rules</title> | |
| <link rel="icon" href="MecCog_logo_transparent.png"> | |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600;700&family=DM+Sans:opsz,wght@9..40,400;9..40,500;9..40,700&display=swap"> | |
| <style> | |
| /* Same system as the main page: white ground, indigo ink, orange accent. */ | |
| :root { | |
| color-scheme: light; | |
| --indigo: #2B1B74; --indigo-soft: #4A35A8; | |
| --amber: #F0802B; --mint: #16A06A; | |
| --page: #FFFFFF; --tint: #F6F5FB; --surface: #FFFFFF; | |
| --ink: #15152B; --ink-soft: #5C5C78; --rule: #E6E5EF; | |
| --display: "Poppins", system-ui, sans-serif; | |
| --body: "DM Sans", system-ui, -apple-system, sans-serif; | |
| } | |
| * { box-sizing: border-box; } | |
| html, body { margin: 0; } | |
| body { background: var(--page); color: var(--ink); font-family: var(--body); font-size: 1.0625rem; line-height: 1.7; } | |
| .in { max-width: 82rem; margin-inline: auto; padding-left: max(20px, env(safe-area-inset-left,0px)); padding-right: max(20px, env(safe-area-inset-right,0px)); } | |
| .nav { border-bottom: 1px solid var(--rule); } | |
| .nav .in { display: flex; align-items: center; gap: 1.5rem; padding-block: 1.1rem; flex-wrap: wrap; } | |
| .nav .logo { display: flex; align-items: center; gap: .65rem; color: var(--ink); text-decoration: none; font-family: var(--display); font-weight: 700; font-size: 1.1rem; } | |
| .nav .logo img { width: 38px; height: 38px; } | |
| .nav menu { display: flex; gap: 1.4rem; list-style: none; margin: 0 0 0 auto; padding: 0; flex-wrap: wrap; } | |
| .nav menu a { color: var(--ink-soft); text-decoration: none; font-size: .92rem; font-weight: 500; } | |
| .nav menu a:hover { color: var(--indigo-soft); } | |
| .btn { display: inline-flex; align-items: center; gap: .5rem; font-family: var(--display); font-weight: 600; font-size: .88rem; padding: .55rem 1.15rem; border-radius: 999px; text-decoration: none; background: var(--amber); color: #231306; border: 2px solid transparent; } | |
| .btn:hover { filter: brightness(1.07); } | |
| header.page { padding-block: 3rem 1rem; } | |
| header.page .eyebrow { font-family: var(--display); font-size: .72rem; font-weight: 600; letter-spacing: .18em; text-transform: uppercase; color: var(--amber); margin: 0 0 .9rem; } | |
| header.page h1 { font-family: var(--display); font-weight: 700; font-size: clamp(2.1rem,1.5rem + 2.6vw,3.2rem); letter-spacing: -0.025em; line-height: 1.12; margin: 0; color: var(--indigo); } | |
| header.page .lede { font-size: 1.15rem; color: var(--ink-soft); max-width: 44ch; margin: 1rem 0 0; } | |
| /* the decision panel */ | |
| .glance { background: var(--tint); border: 1px solid var(--rule); border-radius: 18px; padding: clamp(1.4rem,1rem + 1.6vw,2.2rem); margin-block: 2.5rem 3.5rem; } | |
| .glance h2 { font-family: var(--display); font-size: 1.15rem; font-weight: 600; margin: 0 0 1.4rem; color: var(--indigo); } | |
| .when { display: grid; gap: 1rem; grid-template-columns: repeat(auto-fit,minmax(11rem,1fr)); margin-bottom: 1.8rem; } | |
| .when div { background: var(--surface); border: 1px solid var(--rule); border-radius: 12px; padding: 1rem 1.1rem; min-width: 0; } | |
| .when dt { font-family: var(--display); font-size: .68rem; font-weight: 600; letter-spacing: .14em; text-transform: uppercase; color: var(--ink-soft); } | |
| .when dd { margin: .35rem 0 0; font-family: var(--display); font-weight: 600; font-size: 1.15rem; color: var(--indigo); } | |
| .when dd.go { color: var(--mint); } | |
| .when-note { display: block; margin-top: .2rem; font-family: var(--body); font-size: .78rem; font-weight: 500; line-height: 1.3; color: var(--ink-soft); } | |
| .facts { display: grid; gap: .1rem; } | |
| .facts > div { display: grid; gap: .2rem 1.5rem; padding: .75rem 0; border-top: 1px solid var(--rule); } | |
| @media (min-width: 46rem) { .facts > div { grid-template-columns: 13rem 1fr; align-items: baseline; } } | |
| .facts dt { font-weight: 600; color: var(--ink); font-size: .95rem; } | |
| .facts dd { margin: 0; color: var(--ink-soft); font-size: .97rem; } | |
| .glance:has(+ .glance-note) { margin-bottom: 1rem; } | |
| .glance-note { margin: 0 0 3.5rem; padding-inline: .25rem; color: var(--ink-soft); font-size: .95rem; } | |
| /* phones: the three dates side by side, and labels run into their text as the body's lead-ins do */ | |
| @media (max-width: 45.99rem) { | |
| .when { grid-template-columns: repeat(3, minmax(0, 1fr)); gap: .5rem; margin-bottom: 1.2rem; } | |
| .when div { padding: .65rem .7rem; border-radius: 10px; } | |
| .when dt { font-size: .58rem; letter-spacing: .1em; line-height: 1.35; } | |
| .when dd { margin-top: .25rem; font-size: .95rem; line-height: 1.3; } | |
| .facts > div { display: block; padding: .65rem 0; } | |
| .facts dt { display: inline; } | |
| .facts dt::after { content: ". "; } | |
| .facts dd { display: inline; } | |
| } | |
| main { padding-bottom: 4rem; } | |
| main .in { display: grid; grid-template-columns: 1fr min(54rem, 100%) 1fr; } | |
| main .in > * { grid-column: 2; } | |
| main .in > .tablewrap { grid-column: 1 / -1; } | |
| h1, h2, h3 { font-family: var(--display); line-height: 1.2; text-wrap: balance; } | |
| main h1 { font-size: clamp(1.5rem,1.25rem + 1.3vw,2.1rem); font-weight: 600; color: var(--indigo); margin: 3.25rem 0 1rem; padding-top: 1.75rem; border-top: 2px solid var(--rule); } | |
| main h1:first-child, main .in > .toc + h1 { margin-top: 0; border-top: 0; padding-top: 0; } | |
| main h2 { font-size: 1.25rem; font-weight: 600; margin: 2.25rem 0 .75rem; } | |
| main h3 { font-size: 1.05rem; font-weight: 600; margin: 1.5rem 0 .4rem; } | |
| p { margin: 0 0 1.1rem; } | |
| a { color: var(--indigo-soft); } | |
| strong { font-weight: 700; } | |
| ul, ol { padding-left: 1.3rem; } | |
| li { margin-bottom: .4rem; } | |
| code { font-family: ui-monospace, "SFMono-Regular", Menlo, monospace; font-size: .88em; background: var(--tint); padding: .1rem .35rem; border-radius: 4px; } | |
| main p > strong:first-child { color: var(--indigo); font-family: var(--display); font-weight: 600; } | |
| .tablewrap, table { width: 100%; } | |
| .tablewrap { overflow-x: auto; margin-bottom: 1.5rem; } | |
| table { border-collapse: collapse; font-size: .93rem; min-width: 30rem; } | |
| th, td { padding: .6rem .8rem; border-bottom: 1px solid var(--rule); text-align: left; vertical-align: top; } | |
| th { font-family: var(--display); font-weight: 600; font-size: .78rem; letter-spacing: .06em; text-transform: uppercase; color: var(--ink-soft); border-bottom-width: 2px; } | |
| tr:last-child td { border-bottom: 0; } | |
| footer { background: var(--tint); border-top: 1px solid var(--rule); color: var(--ink-soft); font-size: .92rem; } | |
| footer a { color: var(--indigo-soft); } | |
| /* collapsible top-level sections; each spans the grid so its tables can still break out */ | |
| main .in > .sec { grid-column: 1 / -1; border-top: 2px solid var(--rule); } | |
| .sec > * { max-width: 54rem; margin-inline: auto; } | |
| .sec > .tablewrap { max-width: none; } | |
| .sec > summary { list-style: none; cursor: pointer; display: flex; align-items: center; justify-content: space-between; gap: 1rem; padding: 1.15rem 0; } | |
| .sec > summary::-webkit-details-marker { display: none; } | |
| .sec > summary::after { content: ""; flex: none; width: .55rem; height: .55rem; margin-right: .35rem; border-right: 2px solid var(--ink-soft); border-bottom: 2px solid var(--ink-soft); transform: translateY(-25%) rotate(45deg); } | |
| .sec[open] > summary::after { transform: translateY(25%) rotate(-135deg); } | |
| .sec > summary:hover::after { border-color: var(--indigo-soft); } | |
| .sec > summary h1 { margin: 0; padding: 0; border: 0; } | |
| .sec[open] { padding-bottom: 1.75rem; } | |
| .sec[open] > summary { padding-bottom: .9rem; } | |
| .sec > summary + h2 { margin-top: .5rem; } | |
| .more { border-top: 1px solid var(--rule); border-bottom: 1px solid var(--rule); margin-bottom: 1.1rem; } | |
| .more + .more { margin-top: -1.1rem; border-top: 0; } | |
| .more > summary { list-style: none; cursor: pointer; display: flex; align-items: center; justify-content: space-between; gap: 1rem; padding: .7rem 0; font-family: var(--display); font-weight: 600; font-size: .98rem; color: var(--indigo); } | |
| .more > summary::-webkit-details-marker { display: none; } | |
| .more > summary::after { content: ""; flex: none; width: .45rem; height: .45rem; margin-right: .35rem; border-right: 2px solid var(--ink-soft); border-bottom: 2px solid var(--ink-soft); transform: translateY(-25%) rotate(45deg); } | |
| .more[open] > summary::after { transform: translateY(25%) rotate(-135deg); } | |
| .more > summary:hover::after { border-color: var(--indigo-soft); } | |
| .more[open] { padding-bottom: .3rem; } | |
| .sec-controls { display: flex; justify-content: flex-end; margin-bottom: .75rem; } | |
| .expand-all { font: inherit; font-size: .88rem; font-weight: 500; color: var(--indigo-soft); background: none; border: 1px solid var(--rule); border-radius: 999px; padding: .35rem .95rem; cursor: pointer; } | |
| .expand-all:hover { border-color: var(--indigo-soft); } | |
| main h1[id], main h2[id] { scroll-margin-top: 1.5rem; } | |
| /* contents: collapsed above the text on narrow screens, sticky in the left margin on wide ones */ | |
| .toc { margin: 0 0 2.25rem; border: 1px solid var(--rule); border-radius: 12px; background: var(--tint); font-size: .86rem; line-height: 1.4; } | |
| .toc summary { cursor: pointer; padding: .75rem 1rem; font-family: var(--display); font-weight: 600; font-size: .9rem; color: var(--indigo); } | |
| .toc ol { list-style: none; margin: 0; padding: 0 1rem 1rem; } | |
| .toc ol ol { padding: 0 0 .2rem .8rem; } | |
| .toc li { margin: 0; } | |
| .toc a { display: block; padding: .3rem 0 .3rem .75rem; border-left: 2px solid var(--rule); color: var(--ink); text-decoration: none; } | |
| .toc ol ol a { color: var(--ink-soft); } | |
| .toc a:hover { color: var(--indigo-soft); } | |
| .toc a[aria-current="location"] { color: var(--indigo); border-left-color: var(--indigo); font-weight: 600; } | |
| @media (min-width: 80rem) { | |
| main .in > .sec { grid-column: 2 / -1; } | |
| .sec > * { margin-inline: 0; } | |
| main .in > .toc { grid-column: 1; grid-row: 1 / span 200; align-self: start; position: sticky; top: 1.5rem; max-height: calc(100vh - 3rem); overflow-y: auto; margin: 0 2rem 0 0; border: 0; border-radius: 0; background: none; } | |
| .toc summary { display: none; } | |
| .toc ol { padding: 0; } | |
| } | |
| :focus-visible { outline: 3px solid var(--indigo-soft); outline-offset: 2px; } | |
| </style> | |
| </head> | |
| <body> | |
| <nav class="nav"> | |
| <div class="in"> | |
| <a class="logo" href="index.html"><img src="MecCog_logo_transparent.png" alt=""> MecCog</a> | |
| <menu> | |
| <li><a href="index.html">Home</a></li> | |
| <li><a href="hypotheses.html">Hypotheses</a></li> | |
| <li><a href="rules.html">Rules</a></li> | |
| <li><a href="official-rules.html">Official rules</a></li> | |
| <li><a href="resources.html">Shared resources</a></li> | |
| <li><a href="#the-directness-class">Directness</a></li> | |
| </menu> | |
| <a class="btn" href="hypotheses.html">See the hypotheses</a> | |
| </div> | |
| </nav> | |
| <header class="page"> | |
| <div class="in"> | |
| <p class="eyebrow">Papers and Findings · Step 1</p> | |
| <h1>Challenge rules</h1> | |
| <section class="glance"> | |
| <dl class="when"> | |
| <div><dt>Opens</dt><dd class="go">November 9th, 2026<span class="when-note">00:00 UTC</span></dd></div> | |
| <div><dt>Submission deadline</dt><dd>December 20th, 2026<span class="when-note">end of the day, anywhere on Earth</span></dd></div> | |
| <div><dt>You have</dt><dd>6 weeks</dd></div> | |
| </dl> | |
| <dl class="facts"> | |
| <div><dt>What you do</dt><dd>Pick one of five APOE4 endosome mechanism hypotheses. Find the papers that bear on it, and extract the relevant experimental findings from them.</dd></div> | |
| <div><dt>What you hand in</dt><dd>One spreadsheet per hypothesis, in the official template, 19 columns, passing the validator. A submission that fails validation cannot be uploaded.</dd></div> | |
| <div><dt>How many</dt><dd>Five submissions per hypothesis, per team. You may work on as many of the five hypotheses as you like.</dd></div> | |
| <div><dt>Limits per submission</dt><dd>25 papers</dd></div> | |
| <div><dt>Who can take part</dt><dd>Anyone. There is no separate registration step: your first submission registers your team. A team of one is fine, and every team names a leader, since a team with no leader cannot submit.</dd></div> | |
| <div><dt>What you must provide</dt><dd>A public repository or archive holding a README that describes how your submission was produced, and the commit it was built from. Without it the submission does not validate.</dd></div> | |
| <div><dt>Human or agent</dt><dd>Either, or both, or several agents together. You say which on the Method sheet.</dd></div> | |
| <div><dt>What you get</dt><dd>Every team in good standing is invited to co-author the challenge paper.</dd></div> | |
| <div><dt>When you will hear back</dt><dd>The first round is assessed partly by hand, so results will take a while. That slowness is deliberate: what we learn from judging this round by hand is what lets us automate the judging of the next ones.</dd></div> | |
| <div><dt>What matters most</dt><dd>Accuracy over volume. A short, focused submission scores better than one padded with general Alzheimer's literature, and findings that argue against the hypothesis are wanted, not penalised.</dd></div> | |
| </dl> | |
| </section> | |
| <p class="glance-note">The <a href="official-rules.html">official rules</a> apply to every MecCog challenge, and registering accepts them. Questions go on the <a href="https://huggingface.co/spaces/MecCog/meccog-challenge/discussions" target="_blank" rel="noopener">Community tab</a> of this Space.</p> | |
| </div> | |
| </header> | |
| <main> | |
| <div class="in"> | |
| <details class="toc" open> | |
| <summary>On this page</summary> | |
| <nav aria-label="On this page"> | |
| <ol> | |
| <li><a href="#about-this-challenge">About This Challenge</a> | |
| <ol> | |
| <li><a href="#what-youll-do">What You’ll Do</a></li> | |
| <li><a href="#the-hypotheses">The Hypotheses</a></li> | |
| </ol></li> | |
| <li><a href="#teams">Teams</a> | |
| </li> | |
| <li><a href="#challenge-rules">Challenge Rules</a> | |
| <ol> | |
| <li><a href="#what-we-are-asking-for">What We Are Asking For</a></li> | |
| <li><a href="#where-submissions-go-wrong">Where Submissions Go Wrong</a></li> | |
| </ol></li> | |
| <li><a href="#submission-format">Submission Format</a> | |
| <ol> | |
| <li><a href="#the-directness-class">The directness class</a></li> | |
| <li><a href="#publishing-your-method">Publishing your method</a></li> | |
| <li><a href="#recording-what-the-paper-does-not-report">Recording what the paper does not report</a></li> | |
| </ol></li> | |
| <li><a href="#how-youll-be-evaluated">How You’ll Be Evaluated</a> | |
| </li> | |
| <li><a href="#submitting">Submitting</a> | |
| </li> | |
| <li><a href="#how-to-get-started">How to Get Started</a> | |
| </li> | |
| <li><a href="#timeline-and-questions">Timeline and Questions</a> | |
| </li> | |
| </ol> | |
| </nav> | |
| </details> | |
| <div class="sec-controls" hidden><button type="button" class="expand-all" aria-pressed="false">Expand all</button></div> | |
| <details class="sec"> | |
| <summary><h1 id="about-this-challenge">About This Challenge</h1></summary> | |
| <p>Welcome to the MecCog Challenge, a collaborative effort to map the | |
| mechanisms by which the APOE4 genetic variant increases the risk of | |
| Alzheimer’s disease.</p> | |
| <h2 id="what-youll-do">What You’ll Do</h2> | |
| <p>Your task is to find relevant research papers and extract key | |
| experimental findings that relate to the specific APOE4-Alzheimer’s | |
| hypothesis you select.</p> | |
| <p>For each hypothesis, you will:</p> | |
| <ul> | |
| <li><p>Search the scientific literature (PubMed, preprints, databases, | |
| web sources)</p></li> | |
| <li><p>Identify papers that bear on that mechanism</p></li> | |
| <li><p>Extract experimental findings from those papers (measurements, | |
| effects, experimental conditions)</p></li> | |
| <li><p>Record everything in a structured spreadsheet that we | |
| validate</p></li> | |
| </ul> | |
| <p>This is Step 1 of a multi-phase challenge. Later phases will ask you | |
| to assess evidence, estimate mechanism confidence, and design | |
| experiments. For now we focus on finding and documenting what is already | |
| known.</p> | |
| <h2 id="the-hypotheses">The Hypotheses</h2> | |
| <p>Hypotheses are released in batches, starting November 9th, 2026. The | |
| current batch is listed on the <a href="hypotheses.html">hypotheses page</a>.</p> | |
| <p>Pick the hypothesis you want to work on. Do not catalog Alzheimer’s | |
| mechanisms in general. We need findings that directly test or inform the | |
| one hypothesis you picked, and a submission padded with general | |
| Alzheimer’s literature scores worse than a short, focused one.</p> | |
| <p>Each hypothesis carries qualifying clauses, and they are part of it: a finding in aged or | |
| AD tissue does not test a hypothesis stated for non-aged, non-AD tissue, | |
| and should be recorded with that mismatch noted rather than presented as | |
| direct support.</p> | |
| </details> | |
| <details class="sec"> | |
| <summary><h1 id="teams">Teams</h1></summary> | |
| <p><strong>There is no separate registration step.</strong> Your first submission is what registers your team. The Method sheet carries the team name, the team leader and the leader’s email, and those three cells are what tell us who produced the work and where to write.</p> | |
| <p><strong>Teams</strong>. Every submission belongs to a team. One person is named as the team leader, and a submission with no leader does not validate.</p> | |
| <p>Working alone is fine. Give your team a name and put yourself down as its leader. The team exists so that submissions have an owner and a point of contact, not to require you to find collaborators.</p> | |
| <p><strong>Identity</strong>. Use an academic or registered company email address for the team leader where you have one. If you do not, use whatever address reaches you and say in your README who you are and what you do. We may write before the submission is accepted.</p> | |
| <p>You may belong to more than one team, but creating a second team | |
| requires a written explanation, for example that your group has two | |
| genuinely different methods and wants both assessed. Creating several | |
| teams to multiply your submissions is not allowed.</p> | |
| </details> | |
| <details class="sec" open> | |
| <summary><h1 id="challenge-rules">Challenge Rules</h1></summary> | |
| <h2 id="what-we-are-asking-for">What We Are Asking For</h2> | |
| <p><strong>Search widely.</strong> PubMed, the preprint servers, Google | |
| Scholar, databases, and anything else you think bears on the hypothesis. | |
| We would rather see a source you end up setting aside than miss one that | |
| mattered.</p> | |
| <p><strong>Extract findings precisely.</strong> Each one needs:</p> | |
| <ul> | |
| <li><p>A one-sentence description of what was done and what was | |
| observed</p></li> | |
| <li><p>A direct quote from the paper, or N/A if none is | |
| available</p></li> | |
| <li><p>The experimental system, described as precisely as the paper | |
| allows. Not just “astrocytes” but the source of the cells; not just | |
| “mouse” but the strain</p></li> | |
| <li><p>Where in the paper the finding appears, at panel level. Fig 4C, | |
| not Fig 4</p></li> | |
| <li><p>The statistics, with their provenance (see below)</p></li> | |
| <li><p>The directness class, saying how the experiment lines up with the | |
| hypothesis, and the strength of that link</p></li> | |
| </ul> | |
| <p><strong>Keep the findings that argue against the hypothesis.</strong> | |
| This one matters more to us than any other. A result that contradicts | |
| the hypothesis, or that shows no effect, is evidence, and we need it as | |
| much as we need the supporting ones. Record it like any other finding, | |
| with the same care over the quote and the statistics. The whole reason | |
| this project exists is a literature in which most reported mechanisms | |
| are probably wrong; a submission that quietly drops the inconvenient | |
| results reproduces that problem inside the challenge. The | |
| cross-comparison between submissions tends to show when it has | |
| happened.</p> | |
| <p><strong>Leave no cell empty.</strong> Every cell carries either a | |
| value or the literal N/A. A blank tells us nothing, and we cannot tell | |
| it apart from an oversight.</p> | |
| <p><strong>Get the metadata right.</strong> DOI, PubMed ID where the | |
| source has one, and source type, for every paper. These are what let us | |
| line your submission up against everyone else’s.</p> | |
| <p><strong>Validate before you submit.</strong> Use the Validate button | |
| on the submission page, or run the validator yourself beforehand; the | |
| download link is on the <a href="resources.html">shared resources page</a>. A submission that fails | |
| validation cannot be uploaded, so it is worth running early rather than | |
| on the deadline.</p> | |
| <p><strong>Publish your method.</strong> Name a public repository or | |
| archive holding a README that describes how your submission was | |
| produced, and the commit it was built from. Both go on the Method sheet | |
| of the spreadsheet, described below. A list of findings without a method | |
| is a result without a protocol, which is why a submission without them | |
| does not validate.</p> | |
| <h2 id="where-submissions-go-wrong">Where Submissions Go Wrong</h2> | |
| <p>These are the five we expect to see, and what each one costs.</p> | |
| <p><strong>Padding the sheet.</strong> Include only the papers and | |
| findings that genuinely bear on your hypothesis. A submission swollen | |
| with general Alzheimer’s literature scores worse than a short, focused | |
| one. This is not a licence to drop a finding that argues against the | |
| hypothesis: relevance is about the subject, not the direction of the | |
| result. One submission carries at most 25 papers, and the validator enforces it, so order them by relevance from the start and keep the ones that earn their place.</p> | |
| <p><strong>Fabrication.</strong> Every finding must exist in the paper | |
| at the location you cite. Misattributions are caught, and they | |
| disqualify the submission.</p> | |
| <p><strong>An estimate presented as a reported value.</strong> You may | |
| read an effect size off a bar chart, or count the plotted points to get | |
| a sample size, and we would rather have the estimate than a blank. A P | |
| value is the one thing you may never estimate, for the reason given | |
| under Recording what the paper does not report. But an estimate has to | |
| say so: a leading ~ on the value, and the provenance code | |
| estimated_from_figure. An estimate recorded as though the paper printed | |
| it is a fabrication, and it is the kind we are most likely to catch, | |
| because other submissions will have read the same panel.</p> | |
| <p><strong>Invented identifiers.</strong> DOIs and PubMed IDs must | |
| correspond to real, published sources. A plausible-looking identifier | |
| that resolves to nothing is worse than N/A.</p> | |
| <p><strong>Working ahead.</strong> This is Step 1 only. Assessing | |
| confidence, designing experiments and building mechanism schemas all | |
| come later, and doing them now will not count in your favour.</p> | |
| </details> | |
| <details class="sec" open> | |
| <summary><h1 id="submission-format">Submission Format</h1></summary> | |
| <p>One spreadsheet per hypothesis. Download the official template from | |
| the <a href="resources.html">shared resources page</a>.</p> | |
| <p>Each submission is limited to 25 papers.</p> | |
| <p>Use the hypothesis statement exactly as published, including its | |
| qualifying clauses, and copy it verbatim into cell A2 of the Submission | |
| sheet.</p> | |
| <p>Structure</p> | |
| <p>The template has two sheets that the validator reads, | |
| <strong>Submission</strong> and <strong>Method</strong>, already named. | |
| Do not rename them. A third sheet, How to fill this in, comes last and | |
| is there for you; the validator ignores it.</p> | |
| <p>The Submission sheet:</p> | |
| <ul> | |
| <li><p>Row 1: column labels, provided in the template. All of them must | |
| be present, in the template’s order</p></li> | |
| <li><p>Row 2: the hypothesis in column A. The rest of row 2 stays empty</p></li> | |
| <li><p>Row 3 onward: one block per paper, a paper row followed by one | |
| row per finding from that paper</p></li> | |
| </ul> | |
| <p>The Method sheet holds your team, who leads it and how to reach them, and the link to your method, seven labelled rows described under Publishing your method.</p> | |
| <p>The columns of the Submission sheet</p> | |
| <div class="tablewrap"><table> | |
| <colgroup> | |
| <col style="width: 33%" /> | |
| <col style="width: 33%" /> | |
| <col style="width: 33%" /> | |
| </colgroup> | |
| <thead> | |
| <tr class="header"> | |
| <th style="text-align: left;">Column</th> | |
| <th style="text-align: left;">Content</th> | |
| <th style="text-align: left;">Accepted values</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Hypothesis</td> | |
| <td style="text-align: left;">The hypothesis you selected. Row 2 | |
| only</td> | |
| <td style="text-align: left;">The statement copied verbatim from the | |
| <a href="hypotheses.html">hypotheses page</a>. Empty on every other row</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">DOI</td> | |
| <td style="text-align: left;">The source’s DOI</td> | |
| <td style="text-align: left;">A DOI, or N/A for a source that has | |
| none</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Paper/source type</td> | |
| <td style="text-align: left;">What kind of source it is</td> | |
| <td style="text-align: left;">Exactly one of PubMed published, PubMed | |
| preprint, Web article, Database, Other</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">PubMed ID</td> | |
| <td style="text-align: left;">Required for any source catalogued in | |
| PubMed</td> | |
| <td style="text-align: left;">Digits only, or N/A</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Paper/Finding ID</td> | |
| <td style="text-align: left;">What the row is: a paper or one of its | |
| findings</td> | |
| <td style="text-align: left;">P1, P2 and so on for papers, in order of | |
| relevance, P1 holding the findings most relevant to the hypothesis. | |
| P1.F1, P1.F2 and so on for the findings of P1</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Finding description</td> | |
| <td style="text-align: left;">Plain English, one sentence</td> | |
| <td style="text-align: left;">Free text</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Finding quote</td> | |
| <td style="text-align: left;">The exact text from the paper</td> | |
| <td style="text-align: left;">Free text, or N/A</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Finding summary</td> | |
| <td style="text-align: left;">Structured notation, used to match | |
| findings across submissions</td> | |
| <td style="text-align: left;">For example (APOE3 -> APOE4) -> | |
| (less ABCA1), or N/A where the form does not fit</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Finding relevance</td> | |
| <td style="text-align: left;">How relevant the finding is to the | |
| hypothesis</td> | |
| <td style="text-align: left;">A number from 0 to 1</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Experimental system</td> | |
| <td style="text-align: left;">The system and the conditions</td> | |
| <td style="text-align: left;">Free text, as precise as the paper | |
| allows</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Data location</td> | |
| <td style="text-align: left;">Where the data are</td> | |
| <td style="text-align: left;">Fig 4C, Table 2, Supplementary Fig 3, or | |
| an exact quoted sentence</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Effect size</td> | |
| <td style="text-align: left;">The size of the change</td> | |
| <td style="text-align: left;">A percentage (40%, 300%), a fold change | |
| (2.5x), a standardised effect size as the paper gives it (d=3.7, | |
| r=0.58), any of these with a leading ~ where you estimated it from a | |
| figure (~40%), the word qualitative where the paper shows a direction | |
| you cannot put a number on, or N/A</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Effect size provenance</td> | |
| <td style="text-align: left;">Where that value came from, or why there | |
| is none</td> | |
| <td style="text-align: left;">One of the codes below</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">P value</td> | |
| <td style="text-align: left;">The reported significance</td> | |
| <td style="text-align: left;">A number (0.03), an inequality (<0.001, | |
| >0.05), or N/A</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">P value provenance</td> | |
| <td style="text-align: left;">Where that value came from, or why there | |
| is none</td> | |
| <td style="text-align: left;">One of the codes below</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Sample size</td> | |
| <td style="text-align: left;">The number of samples</td> | |
| <td style="text-align: left;">A whole number (12), a range as the paper | |
| gives it (8-12), either with a leading ~ where you counted plotted | |
| points instead of reading a stated number (~12), or N/A</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Sample size provenance</td> | |
| <td style="text-align: left;">Where that value came from, or why there | |
| is none</td> | |
| <td style="text-align: left;">One of the codes below</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Directness class</td> | |
| <td style="text-align: left;">How directly the finding bears on the | |
| hypothesis</td> | |
| <td style="text-align: left;">A number from 1 to 5, on finding rows | |
| only</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Directness strength</td> | |
| <td style="text-align: left;">How strong that link is</td> | |
| <td style="text-align: left;">A number from 1 to 5 where 1 is the | |
| strongest, or the number with its word (2 strong), on finding rows | |
| only</td> | |
| </tr> | |
| </tbody> | |
| </table></div> | |
| <h2 id="the-directness-class">The directness class</h2> | |
| <p>Every hypothesis in this challenge has the same shape. A change in | |
| one system variable A causes a change in another system variable B. For | |
| the ABCA1 hypothesis, A is the APOE4 variant in place of APOE3, and B is | |
| the abundance of ABCA1 protein in the astrocyte outer membrane.</p> | |
| <p>ABCA1 is used on this page and in the example sheet only as a worked example. It is not one of the five challenge hypotheses.</p> | |
| <p>The directness class says how the experiment behind your finding | |
| lines up with that shape. It describes what the experiment did, not what | |
| the evidence is worth. Use one of:</p> | |
| <div class="tablewrap"><table> | |
| <colgroup> | |
| <col style="width: 50%" /> | |
| <col style="width: 50%" /> | |
| </colgroup> | |
| <thead> | |
| <tr class="header"> | |
| <th style="text-align: left;">Class</th> | |
| <th style="text-align: left;">The experiment</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr class="odd"> | |
| <td style="text-align: left;">1 Direct</td> | |
| <td style="text-align: left;">A itself was perturbed, or varies | |
| naturally, and B itself was measured</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">2 Indirect input</td> | |
| <td style="text-align: left;">A was reached through a stand-in A’, and B | |
| itself was measured</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">3 Indirect output</td> | |
| <td style="text-align: left;">A itself was perturbed, and B was inferred | |
| from a stand-in B’</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">4 Indirect input and output</td> | |
| <td style="text-align: left;">Both ends go through stand-ins</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">5 Other</td> | |
| <td style="text-align: left;">None of the shapes above fit. Say why in | |
| Finding description</td> | |
| </tr> | |
| </tbody> | |
| </table></div> | |
| <details class="more"> | |
| <summary>Worked example: classes 1 to 4</summary> | |
| <p>A worked example, for the hypothesis that APOE4 reduces ABCA1 | |
| abundance in the astrocyte outer membrane. Measuring membrane ABCA1 in | |
| APOE4 and APOE3 astrocytes is class 1: the genotype is A itself and | |
| membrane abundance is B itself. Reducing ABCA1 expression with a | |
| microRNA and then measuring membrane abundance is class 2, because the | |
| microRNA is a stand-in for the genotype. Comparing APOE4 and APOE3 | |
| astrocytes but measuring cholesterol efflux instead of membrane | |
| abundance is class 3, because efflux is a stand-in for B. Adding | |
| recombinant apoE4 protein to a reporter cell line and measuring efflux | |
| is class 4, because both ends are stand-ins.</p> | |
| </details> | |
| <p>Three rules settle most of the hard calls.</p> | |
| <p>A perturbation does not have to be introduced by the experimenter. | |
| Natural variation counts: comparing APOE4 carriers with APOE3 carriers, | |
| or comparing isoforms, is perturbing A. Do not drop a finding to a lower | |
| class because the experimenters observed A rather than manipulated | |
| it.</p> | |
| <p>The class belongs to the finding, not to the paper. One paper can | |
| hold findings of several classes, so the cell is filled on finding rows | |
| and left empty on paper rows. The validator rejects a class on a paper | |
| row.</p> | |
| <p>The class is relative to the hypothesis you selected. The same | |
| experiment would take a different class against a different hypothesis, | |
| so do not carry your classes from one submission to another.</p> | |
| <p>The strength of the link. The class says what shape the experiment | |
| had. The strength, d in the directness document, says how strong the | |
| link is between that experiment and the hypothesis, on a scale of 1 to | |
| 5. Note the direction: <strong>1 is the strongest link and 5 the | |
| weakest</strong>, the same direction as the class, so a low number | |
| always means closer to the hypothesis. The template offers the number | |
| with its word, 1 very strong through 5 very weak, and a bare number is | |
| accepted too.</p> | |
| <p>One question sets it. Could the perturbation have reached the readout | |
| by a route that does not pass through the hypothesis? The directness | |
| document illustrates this with two experiments on a different ABCA1 | |
| hypothesis, that reduced ABCA1 abundance increases Alzheimer’s risk. One | |
| reduces ABCA1 expression with a microRNA and measures risk in a mouse | |
| model: the microRNA’s other targets are known and none of them look | |
| likely to affect risk, so there is no plausible route to risk that goes | |
| around ABCA1, and the link is strong. The other uses an LXR/RXR agonist | |
| to raise ABCA1 and also measures risk: but the agonist changes APOE | |
| expression too, which is an obvious route to risk that never touches | |
| ABCA1 abundance, so the same class of experiment gives a much weaker | |
| link.</p> | |
| <p>From that question:</p> | |
| <div class="tablewrap"><table> | |
| <colgroup> | |
| <col style="width: 50%" /> | |
| <col style="width: 50%" /> | |
| </colgroup> | |
| <thead> | |
| <tr class="header"> | |
| <th style="text-align: left;">Strength</th> | |
| <th style="text-align: left;">When</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr class="odd"> | |
| <td style="text-align: left;">1 very strong</td> | |
| <td style="text-align: left;">Class 1. A itself and B itself, so there | |
| is no stand-in for another route to enter by</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">2 strong</td> | |
| <td style="text-align: left;">No plausible route around the hypothesis, | |
| and the stand-in is well characterised</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">3 moderate</td> | |
| <td style="text-align: left;">A possible route exists and you can argue | |
| against it</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">4 weak</td> | |
| <td style="text-align: left;">A known, obvious route around the | |
| hypothesis</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">5 very weak</td> | |
| <td style="text-align: left;">Several such routes, or the stand-in is | |
| too far from A or B to say</td> | |
| </tr> | |
| </tbody> | |
| </table></div> | |
| <p>Class 4 has a stand-in at both ends, so there are two places an | |
| alternative route can enter, and it rarely does better than 3. A class 5 | |
| finding is judged on the same question and explained in Finding | |
| description.</p> | |
| <p>The validator checks the pair for the one error that is easy to make. | |
| A class 1 finding with a strength of 4 or 5 is flagged, because class 1 | |
| has no stand-in at either end, so that combination usually means the | |
| scale was read upside down.</p> | |
| <h2 id="publishing-your-method">Publishing your method</h2> | |
| <p>A list of findings without a method is a result without a protocol. | |
| This challenge is as much about how the literature was read as about | |
| what was found, so the method has to be public and it has to be | |
| findable.</p> | |
| <p>What to publish. A public repository or archive with a README.md at | |
| its root holding six sections:</p> | |
| <ol type="1"> | |
| <li><p>Who did the work: a person, an agent, a person and an agent, or | |
| several agents working together</p></li> | |
| <li><p>Model and version, if any was used</p></li> | |
| <li><p>How papers were found: sources and queries</p></li> | |
| <li><p>How full text was obtained</p></li> | |
| <li><p>What a human checked, and on what proportion of the | |
| findings</p></li> | |
| <li><p>Anything else worth knowing</p></li> | |
| </ol> | |
| <p>Copy method_README_template.md from the <a href="resources.html">shared resources page</a>. For | |
| work done entirely by hand, six short paragraphs are enough; nobody | |
| expects code that does not exist.</p> | |
| <p>What goes in the spreadsheet. The Method sheet carries seven labelled rows. Fill the cell to the right of each label:</p> | |
| <div class="tablewrap"><table> | |
| <colgroup> | |
| <col style="width: 50%" /> | |
| <col style="width: 50%" /> | |
| </colgroup> | |
| <thead> | |
| <tr class="header"> | |
| <th style="text-align: left;">Label</th> | |
| <th style="text-align: left;">Content</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Team</td> | |
| <td style="text-align: left;">Your team name. Use the same one on every submission you make, across all hypotheses</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Team leader</td> | |
| <td style="text-align: left;">The person the organizers contact about this submission. A team of one names themselves</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Team leader email</td> | |
| <td style="text-align: left;">Where the organizers write about this submission. An institutional or company address where you have one</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Method URL</td> | |
| <td style="text-align: left;">The public repository or archive holding | |
| your README</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Commit SHA</td> | |
| <td style="text-align: left;">The commit your submission was built from, | |
| or N/A for an archive with a DOI</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">Who did the work</td> | |
| <td style="text-align: left;">Exactly one of person, agent, both, | |
| collab</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">Model and version</td> | |
| <td style="text-align: left;">The exact version, one per agent where | |
| several were involved. For work done without a model, none or N/A</td> | |
| </tr> | |
| </tbody> | |
| </table></div> | |
| <p>The four values for who did the work: <strong>person</strong> for a | |
| human working alone, <strong>agent</strong> for a single agent, | |
| <strong>both</strong> for a human and an agent, <strong>collab</strong> | |
| for several agents working together. For collab, list every model in | |
| Model and version, one per agent, and say in section 1 of your README | |
| which agent did which step and whether a person was involved as | |
| well.</p> | |
| <p>The two cells have to agree. A submission made entirely by hand is | |
| person, and its Model and version is none, or N/A if you prefer the | |
| convention used everywhere else in the format. Anything other than | |
| person means a model was used, so the model and its version are | |
| required. The validator checks the pair in both directions.</p> | |
| <p>Where to publish it. Any public host: GitHub, GitLab, Codeberg, or an | |
| archive with a DOI such as Zenodo or OSF. A private repository does not | |
| count, and neither does a link that needs an account to open.</p> | |
| <p>Why the commit SHA. A repository can be rewritten, emptied or made | |
| private after the deadline, so a bare link is not a record of what | |
| produced the submission. The SHA pins it. Run git rev-parse HEAD and put | |
| the result in the Commit SHA cell of the Method sheet. An archive with a | |
| DOI is already immutable, so it takes N/A instead.</p> | |
| <h2 id="recording-what-the-paper-does-not-report">Recording what the | |
| paper does not report</h2> | |
| <p>A missing number can mean two opposite things. Either the paper never | |
| reported it, which is a fact about the literature and one of the things | |
| this challenge exists to measure. Or it was reported and the submission | |
| did not find it, which is a fact about the submission. Writing N/A for | |
| both makes them indistinguishable.</p> | |
| <p>So every statistic carries a provenance code alongside it. Use | |
| exactly one of:</p> | |
| <div class="tablewrap"><table> | |
| <colgroup> | |
| <col style="width: 50%" /> | |
| <col style="width: 50%" /> | |
| </colgroup> | |
| <thead> | |
| <tr class="header"> | |
| <th style="text-align: left;">Code</th> | |
| <th style="text-align: left;">Meaning</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr class="odd"> | |
| <td style="text-align: left;">printed_in_text</td> | |
| <td style="text-align: left;">The value is printed in the text, a table | |
| or a figure legend, and you took it from there</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">visual_only</td> | |
| <td style="text-align: left;">The paper shows it graphically or as | |
| significance stars, with no printed number, and you recorded what it | |
| shows without estimating a value</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">estimated_from_figure</td> | |
| <td style="text-align: left;">There is no printed number and you | |
| measured one off the figure, by bar height, axis position or by counting | |
| plotted points. The value carries a leading ~</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">supplement_only</td> | |
| <td style="text-align: left;">The value is in the supplementary | |
| material, and you took it from there</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">not_found_in_source</td> | |
| <td style="text-align: left;">The paper does not report it anywhere you | |
| could find</td> | |
| </tr> | |
| <tr class="even"> | |
| <td style="text-align: left;">full_text_unavailable</td> | |
| <td style="text-align: left;">You could not reach the full text, so you | |
| cannot say whether it is reported</td> | |
| </tr> | |
| <tr class="odd"> | |
| <td style="text-align: left;">ambiguous_panel</td> | |
| <td style="text-align: left;">The panel you cite reports several | |
| comparisons and it is not clear which one the value belongs to</td> | |
| </tr> | |
| </tbody> | |
| </table></div> | |
| <details class="more"> | |
| <summary>Conventions: effect sizes, estimates, P values, significance stars, paywalls, ranges</summary> | |
| <p>Report the effect as the paper reports it. A percentage, a fold | |
| change and a standardised effect size are all acceptable, and the one to | |
| use is the one the authors used. Converting a Cohen’s d into a | |
| percentage, or a fold change into a percentage, introduces an assumption | |
| that is yours and not theirs. Where the paper gives several, give the | |
| one that belongs to the comparison your finding is about.</p> | |
| <p>Estimates are welcome, as long as they say so. Where a figure carries | |
| the information but no number is printed, measure it: an effect size | |
| from the bars, a sample size by counting the plotted points. Write the | |
| value with a leading ~ and use estimated_from_figure. This is better | |
| than N/A, because a marked estimate can be used or discarded at will | |
| while a blank cannot be recovered. Where you cannot put a defensible | |
| number on what the figure shows, use qualitative with visual_only rather | |
| than inventing a precision you do not have.</p> | |
| <p>Never estimate a P value. An effect size and a sample size are | |
| quantities the figure displays, so measuring them is reading, with an | |
| error you have declared by writing the ~. A P value is not displayed. | |
| The figure shows the data a test was run on; the test itself happened | |
| elsewhere, and its result is either stated or it is not.</p> | |
| <p>So a figure gives you a P value only where the panel or its legend | |
| states one. If the panel carries stars, record what the legend says they | |
| mean. If it carries nothing, the value is N/A with not_found_in_source. | |
| The validator rejects estimated_from_figure on a P value.</p> | |
| <p>Significance stars. Record the inequality the paper’s own legend | |
| gives them, with provenance visual_only. The usual convention is * for | |
| p<0.05, ** for p<0.01 and *** for p<0.001, but that is only the | |
| usual one and it is an example, not the rule: some papers start at ** | |
| for p<0.01, some add **** for p<0.0001, and some use the stars for | |
| an adjusted P value. Read the legend of the figure you are citing and follow it. Where you can read the key, record the inequality, with provenance visual_only, and do not add a ~: the inequality is what the paper asserts, not something you measured. Where the figure gives no key at all, there is no inequality to record, so the value is N/A with provenance ambiguous_panel. What you must not do is write N/A on a panel whose key you could read.</p> | |
| <p>Paywalls. full_text_unavailable means the full text was genuinely out | |
| of reach. Before using it, try the DOI directly, the publisher’s own | |
| page, Europe PMC, PMC, and Unpaywall. “My tool could not fetch it” is | |
| not the same as “it is not available”, and the two are scored | |
| differently.</p> | |
| <p>Ranges and per-group values. Where a paper gives a range (8-12 mice | |
| per group), record the range. Do not pick one end of it: a single number | |
| that is not in the paper is less faithful than the range that is.</p> | |
| </details> | |
| </details> | |
| <details class="sec"> | |
| <summary><h1 id="how-youll-be-evaluated">How You’ll Be Evaluated</h1></summary> | |
| <p>Unlike most community challenges to date, this one has no ground truth | |
| against which to assess the results. Thus, the performance assessment | |
| methods break new ground and may evolve as the project progresses. | |
| Community challenges differ in the extent to which they predefine | |
| assessment criteria. Some, such as those run by Kaggle, predefine a single | |
| metric that will be used for ranking. Others, such as CASP, predefine a | |
| subset of criteria, but encourage independent assessors to introduce new | |
| ones, once the results are in hand. This challenge sets out a set of | |
| initial considerations (below), without an explicit ranking formula. These | |
| may be modified and added to once the results are analyzed.</p> | |
| <p>Initial assessment will be as follows:</p> | |
| <ol type="1"> | |
| <li><p>Cross-evaluation by participants to find the subset of strongest | |
| performing groups (each participant provides subjective scores for the | |
| methods and results submitted by ten other participants, with comments, | |
| generating ten assessments per group). Necessary only if there are a | |
| large number of submissions.</p></li> | |
| <li><p>Comparison of results across groups: which papers were included, | |
| which experiments were extracted from those papers. This does not imply | |
| that findings reported by one or a few groups will be considered | |
| incorrect. Rather, it will help identify ‘easy’ results.</p></li> | |
| <li><p>Evaluation of each finding’s relevance using the evidence | |
| directness criteria.</p></li> | |
| <li><p>Checking of each Finding’s specifics (effect size, P value, | |
| experimental system) against the source paper. (Experience with the | |
| Hugging Face collab showed that providing these specifics is hard for | |
| current methods)</p></li> | |
| <li><p>Where necessary, manual inspection.</p></li> | |
| </ol> | |
| </details> | |
| <details class="sec"> | |
| <summary><h1 id="submitting">Submitting</h1></summary> | |
| <p>Accepted submissions are given an acceptance number, shown on screen. Keep it. Queries about a | |
| submission cannot be handled without its acceptance number.</p> | |
| </details> | |
| <details class="sec"> | |
| <summary><h1 id="how-to-get-started">How to Get Started</h1></summary> | |
| <ol type="1"> | |
| <li><p>Read the <a href="hypotheses.html">hypotheses page</a> and pick one</p></li> | |
| <li><p>Download submission_template.xlsx and method_README_template.md | |
| from <a href="resources.html">shared resources</a></p></li> | |
| <li><p>Search for papers</p></li> | |
| <li><p>Extract findings and fill the spreadsheet as you go, provenance | |
| codes and directness included</p></li> | |
| <li><p>Validate</p></li> | |
| <li><p>Publish your method README, fill in the Method sheet, then upload | |
| the spreadsheet and keep your acceptance number</p></li> | |
| </ol> | |
| </details> | |
| <details class="sec"> | |
| <summary><h1 id="timeline-and-questions">Timeline and Questions</h1></summary> | |
| <ul> | |
| <li><p>Step 1 opens: 00:00 UTC on November 9th, 2026</p></li> | |
| <li><p>First batch of hypotheses released: November 9th, 2026</p></li> | |
| <li><p>Step 1 submission deadline: the end of December 20th, 2026, anywhere on Earth</p></li> | |
| <li><p>Later phases, evidence assessment, mechanism confidence and | |
| experimental design: dates to be announced</p></li> | |
| </ul> | |
| <p>Questions go to the <a href="https://huggingface.co/spaces/MecCog/meccog-challenge/discussions" target="_blank" rel="noopener">Community tab</a> first, then to the organizers. Organizers | |
| and the community are here to help.</p> | |
| <p>Thank you for contributing to MecCog. Together we are building a map | |
| of APOE4 mechanisms.</p> | |
| </details> | |
| </div> | |
| </main> | |
| <footer><div class="in" style="padding-block:2.5rem"><p>Talk to us. The <a href="https://huggingface.co/spaces/MecCog/meccog-challenge/discussions" target="_blank" rel="noopener">Community tab</a> of this Space is where questions get asked and answered, and it is usually the fastest route, since someone has often been there before you. Questions about the format especially tend to be everyone’s.</p><p><a href="index.html">Back to the main page</a>.</p></div></footer> | |
| <script> | |
| (function () { | |
| var toc = document.querySelector(".toc"); | |
| var secs = Array.prototype.slice.call(document.querySelectorAll("details.sec, details.more")); | |
| var wide = window.matchMedia("(min-width: 80rem)"); | |
| // open the section holding an element, so links and anchors never land on hidden text | |
| function reveal(el) { | |
| var d = el && el.closest ? el.closest("details.sec, details.more") : null; | |
| while (d) { if (!d.open) d.open = true; d = d.parentElement ? d.parentElement.closest("details.sec, details.more") : null; } | |
| } | |
| // expand all / collapse all | |
| var ctl = document.querySelector(".sec-controls"), btn = ctl && ctl.querySelector(".expand-all"); | |
| function label() { | |
| if (!btn) return; | |
| var all = secs.every(function (d) { return d.open; }); | |
| btn.textContent = all ? "Collapse all" : "Expand all"; | |
| btn.setAttribute("aria-pressed", all ? "true" : "false"); | |
| } | |
| if (btn) { | |
| ctl.hidden = false; | |
| btn.addEventListener("click", function () { | |
| var openAll = !secs.every(function (d) { return d.open; }); | |
| secs.forEach(function (d) { d.open = openAll; }); | |
| label(); | |
| }); | |
| } | |
| secs.forEach(function (d) { d.addEventListener("toggle", function () { label(); queue(); }); }); | |
| window.addEventListener("beforeprint", function () { secs.forEach(function (d) { d.open = true; }); }); | |
| label(); | |
| function fromHash() { if (location.hash) { var el = document.getElementById(decodeURIComponent(location.hash.slice(1))); if (el) { reveal(el); el.scrollIntoView(); setTimeout(function () { current = -1; update(); }, 0); } } } | |
| if (!toc) return; | |
| function sync() { if (wide.matches) toc.open = true; } | |
| sync(); | |
| if (wide.addEventListener) wide.addEventListener("change", sync); | |
| var links = Array.prototype.slice.call(toc.querySelectorAll("a")); | |
| var heads = links.map(function (a) { return document.getElementById(a.getAttribute("href").slice(1)); }); | |
| var current = -1, queued = false; | |
| function update() { | |
| queued = false; | |
| var line = window.innerHeight * 0.3, idx = 0; | |
| for (var i = 0; i < heads.length; i++) { | |
| var h = heads[i]; | |
| if (!h || h.getClientRects().length === 0) continue; // inside a closed section | |
| if (h.getBoundingClientRect().top <= line) idx = i; | |
| } | |
| if (window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - 2) { | |
| for (var j = heads.length - 1; j >= 0; j--) { if (heads[j] && heads[j].getClientRects().length) { idx = j; break; } } | |
| } | |
| if (idx === current) return; | |
| current = idx; | |
| links.forEach(function (a, i) { if (i === idx) a.setAttribute("aria-current", "location"); else a.removeAttribute("aria-current"); }); | |
| } | |
| function queue() { if (!queued) { queued = true; requestAnimationFrame(update); } } | |
| window.addEventListener("scroll", queue, { passive: true }); | |
| window.addEventListener("resize", queue); | |
| links.forEach(function (a, i) { | |
| a.addEventListener("click", function () { | |
| reveal(heads[i]); | |
| if (!wide.matches) toc.open = false; | |
| }); | |
| }); | |
| window.addEventListener("hashchange", fromHash); | |
| fromHash(); | |
| update(); | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |