Add model card and forecast script

README.md

@@ -18,13 +18,13 @@ The model does not write an answer and does not abstain. It reads the probabilit
 
 ## Why use it
 
-**The marginal cost is about zero.** The weights sit on your own GPU. A comparable run of hosted models on the same kind of workload cost about $0.025 per call, and that bill grows with volume. A local forward pass does not.
+**The marginal cost is about zero.** The weights sit on your own GPU. A forward pass does not incur a per-call fee.
 
-**One forward pass, no generated text.** Measured batch inference is 0.12–1.6 seconds per question. There is no sampling and no token-by-token decode. Hosted forecasters typically reason in long form and only then yield a probability — one typical call in the same comparison emitted 618 tokens before the number.
+**One forward pass, no generated text.** Measured batch inference is 0.12–1.6 seconds per question. There is no sampling and no token-by-token decode. The probability is read from the next-token distribution.
 
-**No third-party relay.** Scoring does not depend on an outside API, a proxy, or someone else's rate limit. In the comparison run, hosted endpoints dropped coverage and failed calls. This model answered every item it was given.
+**No third-party service.** Scoring does not depend on an outside API, a proxy, or a rate limit. The model answered every item in the evaluation.
 
-**The weights are yours to keep training.** A LoRA update followed by a merge left yes/no calibration error at 0.001–0.003. On 15 other checked items, the answers were unchanged after the merge. A closed model can only be waited on.
+**The weights can be trained further.** A LoRA update followed by a merge left yes/no calibration error at 0.001–0.003. On 15 other checked items, the answers were unchanged after the merge.
 
 ## Model details
 
@@ -51,58 +51,26 @@ The model does not write an answer and does not abstain. It reads the probabilit
 
 ## Usage
 
-The prompt carries the as-of date, the evidence, the question, and the resolution rule. One forward pass reads P(YES) from the next-token logits. No tokens are generated.
+`forecast.py` loads the weights once. Each call returns only the calibrated probability.
 
 ```python
-import torch
-from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration
-
-model_id = "YarrowLab/yarrow-forecast"
-
-model = Qwen3_5ForConditionalGeneration.from_pretrained(
-    model_id,
-    dtype="auto",
-    device_map="auto",
-)
-processor = AutoProcessor.from_pretrained(model_id)
-tokenizer = processor.tokenizer
-
-messages = [
-    {
-        "role": "user",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "Forecast as of 2026-09-05.\n\n"
-                    "EVIDENCE:\n"
-                    "[E1] ...\n\n"
-                    "Question: Will candidate X win the nomination?\n"
-                    "Resolution: the nominee is the candidate who wins the party's nomination.\n"
-                    "Answer YES or NO."
-                ),
-            }
-        ],
-    }
-]
-
-inputs = processor.apply_chat_template(
-    messages,
-    tokenize=True,
-    add_generation_prompt=True,
-    return_dict=True,
-    return_tensors="pt",
+from forecast import Forecaster
+
+model = Forecaster("YarrowLab/yarrow-forecast")
+p = model.forecast(
+    title="Will candidate X win the nomination?",
+    rules="Resolves YES if candidate X is declared the nominee.",
+    evidence="[E1] ...",
+    as_of="2026-09-05",
 )
-inputs = inputs.to(model.device)
-
-with torch.no_grad():
-    logits = model(**inputs).logits[0, -1]
+print(p)  # 0.71
+```
 
-yes_id = tokenizer.encode("YES", add_special_tokens=False)
-no_id = tokenizer.encode("NO", add_special_tokens=False)
-pair = torch.stack([logits[yes_id[0]], logits[no_id[0]]])
-p_yes = torch.softmax(pair, dim=0)[0].item()
-print(p_yes)  # probability in [0, 1]
+```python
+ps = model.forecast_batch([
+    {"title": "...", "rules": "...", "evidence": "...", "as_of": "2026-09-05"},
+    {"title": "...", "rules": "...", "evidence": "...", "as_of": "2026-09-05"},
+])
 ```
 
 ## Evaluation

forecast.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Yes/no forecast. One forward pass, then the fitted map. Returns P(YES) only."""
+from __future__ import annotations
+
+import math
+import string
+
+SYSTEM_PROMPT = "Apply the criterion to the evidence. Choose exactly one listed option. Answer with its letter only."
+THINK_OFF = "<think>\n\n</think>\n\n"
+TEMPERATURE = 1.421
+# Fitted on the primary-election evaluation.
+A = 1.099406044975045
+C = 1.3783573144958756
+
+
+def _logit(p: float) -> float:
+    p = min(max(p, 1e-6), 1 - 1e-6)
+    return math.log(p / (1 - p))
+
+
+def _sigmoid(x: float) -> float:
+    return 1.0 / (1.0 + math.exp(-x))
+
+
+def _letter_ids(tokenizer) -> list[int]:
+    probe = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": "Evidence:\n(none)\n\nQuestion: q\nOptions:\nA) a\nB) b"},
+    ]
+    text = tokenizer.apply_chat_template(probe, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+    if not text.endswith(THINK_OFF):
+        raise ValueError("chat template is missing the expected end marker")
+    base = tokenizer.encode(text, add_special_tokens=False)
+    ids = []
+    for letter in string.ascii_uppercase:
+        piece = tokenizer.encode(letter, add_special_tokens=False)
+        if len(piece) != 1 or tokenizer.encode(text + letter, add_special_tokens=False) != base + piece:
+            raise ValueError(f"{letter!r} is not a single token for this tokenizer")
+        ids.append(piece[0])
+    return ids
+
+
+class Forecaster:
+    """Load once. forecast(...) returns a float in [0, 1]."""
+
+    def __init__(self, model_path: str = "YarrowLab/yarrow-forecast"):
+        import torch
+        from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration
+
+        self._torch = torch
+        self.tok = AutoTokenizer.from_pretrained(model_path)
+        self.model = Qwen3_5ForConditionalGeneration.from_pretrained(
+            model_path, dtype="auto", device_map="auto",
+        ).eval()
+        self._yes, self._no = _letter_ids(self.tok)[:2]
+
+    def forecast(self, title: str, rules: str, evidence: str = "", as_of: str | None = None) -> float:
+        return self.forecast_batch([{
+            "title": title, "rules": rules, "evidence": evidence, "as_of": as_of,
+        }])[0]
+
+    def forecast_batch(self, questions: list[dict]) -> list[float]:
+        return [self._one(q) for q in questions]
+
+    def _one(self, q: dict) -> float:
+        as_of = q.get("as_of") or "the question date"
+        state = f"Forecast as of {as_of}."
+        if q.get("evidence"):
+            state += "\n\nEVIDENCE AVAILABLE AT FORECAST TIME:\n" + q["evidence"]
+        question = (
+            f"{q['title']}\n\nResolution rules (verbatim): {q['rules']}\n\n"
+            "Will this resolve YES, based only on the evidence above?"
+        )
+        state = state if state.strip() else "(none)"
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": "Evidence:\n" + state + "\n\nQuestion: " + question.strip() + "\nOptions:\nA) yes\nB) no"},
+        ]
+        text = self.tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
+        if not text.endswith(THINK_OFF):
+            raise ValueError("chat template is missing the expected end marker")
+        ids = self.tok.encode(text, add_special_tokens=False)
+        input_ids = self._torch.tensor([ids], device=self.model.device)
+        with self._torch.no_grad():
+            logits = self.model(input_ids=input_ids).logits[0, -1].float()
+        pair = self._torch.stack([logits[self._yes], logits[self._no]])
+        yes = self._torch.softmax(pair / TEMPERATURE, dim=0)[0].item()
+        return _sigmoid(A * _logit(yes) + C)