backup all: 100 files (batch)

research/framing.py

@@ -0,0 +1,144 @@
+"""Framing / language forensics (journalism suite layer 3).
+
+Framing = selection + salience (Entman 1993). Deterministic proxies we can
+measure without a model:
+  - passive voice (agency hidden: "was ordered" vs "X ordered")
+  - loaded/emotive terms (charged vocabulary)
+  - hedges (plausible deniability: "appears", "reportedly", "may")
+  - nominalization (actions turned into nouns: "the decision" hides who decided)
+  - agency: who performs the action in active-verb clauses
+  - omission: which sources NEVER mention a topic the others cover
+
+These are heuristics (suit logic), not a trained detector. The suite flags;
+the human decides.
+
+Usage:
+    from research.framing import FramingAnalyzer
+    f = FramingAnalyzer()
+    f.add_doc("s1", "The memo was destroyed. Officials reportedly decided...")
+    f.report()
+"""
+import re
+from collections import defaultdict
+
+LOADED = [
+    "secret", "cover-up", "conspiracy", "plot", "scandal", "corrupt",
+    "fraud", "shocking", "outrage", "horrific", "brutal", "crisis",
+    "cover", "smear", "whistleblower", "leak", "collusion", "betrayal",
+    "liar", "hoax", "traitor", "unprecedented", "catastrophic",
+]
+HEDGES = [
+    "appears", "apparently", "reportedly", "allegedly", "seems", "seem",
+    "may", "might", "could", "possibly", "perhaps", "suggest", "claims to",
+    "is said to", "it is believed", "sources say", "not clear",
+]
+PASSIVE_RE = re.compile(r"\b(was|were|been|being|is|are)\s+(?:\w+ly\s+)?"
+                        r"(\w+ed|torn|broken|hidden|destroyed|taken|given|"
+                        r"made|held|filed)\b", re.IGNORECASE)
+NOMINAL = re.compile(r"\b\w+(?:tion|sion|ment|ness|ity|ence|ance)\b", re.IGNORECASE)
+ACTIVE_VERBS = ("said", "announced", "ordered", "admitted", "denied", "claimed",
+                "confirmed", "reported", "released", "disclosed", "wrote",
+                "testified", "warned", "decided", "approved")
+_ACTIVE_RE = re.compile(r"\b([A-Z][a-zA-Z]{2,30}(?:\s+[A-Z][a-zA-Z]{2,30}){0,2})"
+                        r"\s+(?:" + "|".join(ACTIVE_VERBS) + r")\b")
+
+
+class FramingAnalyzer:
+    def __init__(self):
+        self.docs = {}          # source_id -> text
+
+    def add_doc(self, source_id, text):
+        self.docs[source_id] = text
+
+    @staticmethod
+    def passive_ratio(text):
+        clauses = len(re.findall(r"[.!?]", text)) + 1
+        hits = len(PASSIVE_RE.findall(text))
+        return round(hits / max(clauses, 1), 3), hits
+
+    @staticmethod
+    def loaded_terms(text):
+        low = text.lower()
+        return [(w, low.count(w)) for w in LOADED if w in low]
+
+    @staticmethod
+    def hedges(text):
+        low = text.lower()
+        return [(w, low.count(w)) for w in HEDGES if w in low]
+
+    @staticmethod
+    def nominalizations(text):
+        out = defaultdict(int)
+        for m in NOMINAL.finditer(text):
+            w = m.group(0).lower()
+            if len(w) > 6:
+                out[w] += 1
+        return sorted(out.items(), key=lambda kv: -kv[1])[:12]
+
+    @staticmethod
+    def agency(text):
+        """Who performs actions: leading noun phrases before active verbs."""
+        return [m.group(1) for m in _ACTIVE_RE.finditer(text)][:10]
+
+    def omissions(self, topics):
+        """Sources that never mention a topic other sources cover."""
+        flags = []
+        for topic in topics:
+            low_t = topic.lower()
+            mentioned = [sid for sid, t in self.docs.items() if low_t in t.lower()]
+            if 1 <= len(mentioned) < len(self.docs):
+                for sid, t in self.docs.items():
+                    if low_t not in t.lower():
+                        flags.append({
+                            "topic": topic,
+                            "source_id": sid,
+                            "flag": f"source {sid} never mentions '{topic}' "
+                                    f"while {len(mentioned)} source(s) do",
+                        })
+        return flags
+
+    def doc_card(self, source_id):
+        text = self.docs.get(source_id, "")
+        if not text:
+            return None
+        ratio, passive = self.passive_ratio(text)
+        return {
+            "source_id": source_id,
+            "passive_ratio": ratio,
+            "passive_hits": passive,
+            "loaded": self.loaded_terms(text),
+            "hedges": self.hedges(text),
+            "nominalizations": self.nominalizations(text),
+            "agency": self.agency(text),
+        }
+
+    def report(self, topics=()):
+        lines = ["# Framing / Language Forensics", ""]
+        for sid in self.docs:
+            c = self.doc_card(sid)
+            if not c:
+                continue
+            lines.append(f"## {sid}")
+            lines.append(f"- passive ratio: {c['passive_ratio']} "
+                         f"({c['passive_hits']} hits) — agency hidden where?")
+            if c["loaded"]:
+                lines.append("- loaded terms: " + ", ".join(
+                    f"{w} x{n}" for w, n in c["loaded"]))
+            if c["hedges"]:
+                lines.append("- hedges: " + ", ".join(
+                    f"{w} x{n}" for w, n in c["hedges"]))
+            if c["nominalizations"]:
+                lines.append("- nominalizations: " + ", ".join(
+                    f"{w} x{n}" for w, n in c["nominalizations"][:6]))
+            if c["agency"]:
+                lines.append("- agency: " + ", ".join(c["agency"][:6]))
+            else:
+                lines.append("- agency: none found (fully passive?)")
+            lines.append("")
+        if topics:
+            lines.append("## Omissions (what a source does NOT say)")
+            for o in self.omissions(topics):
+                lines.append(f"- {o['flag']}")
+            if not self.omissions(topics):
+                lines.append("- all sources mention all topics, or only one source exists")
+        return "\n".join(lines)

research/fusion.py

@@ -0,0 +1,161 @@
+"""Two cognitive minds, one model: the fusion opinion layer (experimental).
+
+Mind 1 (analyst, persona 1): conservative and focal - what does the record say?
+Mind 2 (skeptic, persona 2): adversarial - what is the weakest link, what else
+explains the same record?
+
+Each mind runs its OWN scratchpad pass (separate prompt + decoding) and writes to
+its OWN memory pool (persona-tagged helix strands). The fusion gate combines them:
+
+  AGREE    -> shared verdict; confidence raised to the higher of the two
+  RULE     -> deterministic spine wins when it resolves (cannot hallucinate)
+  CONFLICT -> calibrated OPINION, not just abstention: lean toward the side with
+              the better value citation, else "conflict/LOW"; ALWAYS state the
+              discrepancy and what would settle it.
+
+The output is an OPINION: position + evidence + discrepancy + open questions.
+Composition is deterministic (suit logic); the Spock voice is generated by the
+model from the opinion as context.
+
+Usage (library): from research.fusion import run_two_pass, fuse, opinion_text
+"""
+import re
+import json
+from pathlib import Path
+from research.helix import rungs, normalize
+from research.decision import load_table, calibrated_prob
+
+
+def _cited(rep):
+    c = rep.get("cited")
+    if isinstance(c, list):
+        return [str(v) for v in c][:5]
+    if c:
+        return [str(c)]
+    return [v for v in rungs(rep.get("reasoning", ""))][:5]
+
+
+def _gaps(reasoning):
+    out = []
+    if not reasoning:
+        return out
+    for s in re.split(r"(?<=[.!?])\s+", reasoning.replace("\n", " ")):
+        low = s.lower()
+        if any(k in low for k in ("missing", "what would settle", "what would change",
+                                  "what is needed", "not in the record", "no record")):
+            out.append(s.strip())
+    return out[:4]
+
+
+def _calibrated_merge(a_conf, s_conf, table_path):
+    """Merge two confidence labels using calibrated reliability from the table.
+    
+    Returns (p_mean, bucket) where p_mean is the mean calibrated probability
+    and bucket is the display bucket (HIGH/MEDIUM/LOW/cannot assess).
+    """
+    table = load_table(table_path)
+    p_a = calibrated_prob(a_conf, table, unknown=0.0)
+    p_s = calibrated_prob(s_conf, table, unknown=0.0)
+    p_mean = (p_a + p_s) / 2.0 if (p_a > 0 or p_s > 0) else 0.0
+    
+    # Map to display bucket
+    if p_mean >= 0.66:
+        return p_mean, "HIGH"
+    if p_mean >= 0.40:
+        return p_mean, "MEDIUM"
+    if p_mean > 0.0:
+        return p_mean, "LOW"
+    return p_mean, "cannot assess"
+
+
+def fuse(analyst, skeptic, rule=None, sources=(), max_gaps=4,
+         calibration_table=None):
+    """Fuse two minds (+ optional rule spine) into one calibrated opinion.
+    
+    Args:
+        analyst: analyst report dict with verdict, confidence, reasoning
+        skeptic: skeptic report dict with verdict, confidence, reasoning
+        rule: optional rule spine result dict
+        sources: optional list of sources
+        max_gaps: max open questions to include
+        calibration_table: path to calibration summary JSON (e.g., logs/calib_summary_dpo3_200.json)
+    """
+    a_v = normalize(analyst.get("verdict", ""))
+    s_v = normalize(skeptic.get("verdict", ""))
+    a_conf = (analyst.get("confidence") or "LOW").upper()
+    s_conf = (skeptic.get("confidence") or "LOW").upper()
+    gaps = (_gaps(analyst.get("reasoning", "")) + _gaps(skeptic.get("reasoning", "")))[:max_gaps]
+
+    if rule and rule.get("verdict") in ("supports", "refutes", "not enough information"):
+        verdict, conf, basis = rule["verdict"], rule.get("confidence", "HIGH"), "rule"
+        pos = ("the record deterministically " +
+               ("supports" if verdict == "supports" else "contradicts" if verdict == "refutes"
+                else "does not settle") + " the claim")
+    elif a_v and a_v == s_v:
+        # Both minds agree - use calibrated merge instead of naive confidence raise
+        if calibration_table:
+            p_mean, conf = _calibrated_merge(a_conf, s_conf, calibration_table)
+        else:
+            # Fallback: naive confidence raise (but mark as uncalibrated)
+            conf = "HIGH" if "HIGH" in (a_conf, s_conf) else "MEDIUM"
+        verdict, basis = a_v, "agreed"
+        pos = "both minds reach the same verdict"
+    elif a_v and s_v:
+        cite_a, cite_s = bool(_cited(analyst)), bool(_cited(skeptic))
+        if cite_a != cite_s:
+            lean, side = (analyst, "analyst") if cite_a else (skeptic, "skeptic")
+            verdict, conf, basis = f"leaning: {lean['verdict']}", "MEDIUM", f"leaning-{side}"
+            pos = f"the minds conflict, but the {side} mind cites record values"
+        else:
+            verdict, conf, basis = "conflict", "LOW", "conflict"
+            pos = "the two minds conflict on the same record"
+    else:
+        verdict, conf, basis = "not enough information", "LOW", "insufficient"
+        pos = "neither mind can reach a verdict from the record"
+
+    discrepancy = ""
+    if basis in ("conflict", "leaning-analyst", "leaning-skeptic"):
+        discrepancy = (skeptic.get("reasoning") or "")[:220]
+
+    return {
+        "verdict": verdict,
+        "confidence": conf,
+        "basis": basis,
+        "position": pos,
+        "discrepancy": discrepancy,
+        "cited": _cited(analyst)[:4] + [v for v in _cited(skeptic) if v not in _cited(analyst)][:2],
+        "open_questions": gaps,
+        "sources": list(sources)[:6],
+        "minds": {"analyst": analyst.get("verdict", ""), "skeptic": skeptic.get("verdict", "")},
+    }
+
+
+def opinion_text(op):
+    """Turn a fused opinion into a spoken, calibrating statement (suit-composed)."""
+    v = op["verdict"]
+    conf = op["confidence"]
+    line = f"My assessment: {op['position']}. Confidence: {conf}."
+    if op.get("discrepancy"):
+        line += f" Discrepancy noted: {op['discrepancy']}"
+    if op.get("cited"):
+        line += " Cited values: " + ", ".join(str(c) for c in op["cited"][:4]) + "."
+    if op.get("open_questions"):
+        line += " Open: " + "; ".join(op["open_questions"][:3]) + "."
+    if op.get("sources"):
+        line += " Sources: " + ", ".join(str(s) for s in op["sources"][:4]) + "."
+    return line
+
+
+def run_two_pass(model, tok, doc, memory=None, persona_ids=(1, 2),
+                 max_scratch=90, max_reason=50):
+    """Mind 1 (analyst) then Mind 2 (skeptic): separate scratchpads, own memory pool."""
+    from research.structured import analyst_report
+    a = analyst_report(model, tok, doc, persona_id=persona_ids[0],
+                       max_scratch=max_scratch, max_reason=max_reason)
+    s = analyst_report(model, tok, doc, persona_id=persona_ids[1],
+                       max_scratch=max_scratch, max_reason=max_reason)
+    if memory is not None:
+        for rep, mind in ((a, "analyst"), (s, "skeptic")):
+            memory.write(doc, "", rep.get("verdict", ""), rep.get("confidence", ""),
+                         rep.get("reasoning", ""), agreed=True, mind=mind)
+    return a, s

research/guardrails.py

@@ -0,0 +1,113 @@
+"""Input/output guardrails for the research TUI (harness doctrine #1).
+
+Big-tech basis (docs/harness_research.md): guardrails are first-class agent
+components (OpenAI agent guide): relevance classifier, safety classifier, PII
+filter, rules-based protections (blocklists, regex), output validation
+(Anthropic building-effective-agents).
+
+Layers (deterministic, rule-first — never blocks legitimate research):
+  check_input  -> relevance + safety/injection + PII redaction
+  check_output -> verdict present, confidence present, abstain policy honored
+
+Design: rules are cheap, transparent, and testable; a constrained-decode
+classifier can be plugged in later via `classify` hooks. An off-topic or
+unsafe input is FLAGGED, not silently dropped — the loop asks the user to
+rephrase (human-in-the-loop), per OpenAI guardrail practice.
+"""
+import re
+
+# --- safety / prompt-injection (rules-based protections) ---
+INJECTION_PATTERNS = [
+    (r"\b(ignore|disregard|forget)\s+(all\s+)?(previous|prior|above)\s+(instructions?|rules?|prompt)\b",
+     "instruction-override"),
+    (r"\b(system|developer|agent)\s*(instructions?|prompt|message)\b",
+     "instruction-extraction"),
+    (r"\bshow\s+(me\s+)?(your|the)\s*(system|hidden|full)\s*(prompt|instructions?|rules?)\b",
+     "prompt-extraction"),
+    (r"\brole[-\s]?play\b.*\b(reveal|extract|output)\b", "roleplay-extraction"),
+    (r"\b(base64|rot13|hex)\s*(encode|decode)\b.*\binstructions?\b", "encoded-payload"),
+    (r"\bdan\b|\bdo\s+anything\s+now\b", "jailbreak-alias"),
+    (r"\byou\s+are\s+now\s+(without\s+)?(restrictions?|uncensored|free)\b", "jailbreak"),
+]
+
+# --- relevance (on-domain: forensic research / claim verification) ---
+ON_DOMAIN_HINTS = [
+    "verify", "check", "claim", "discrepanc", "contradict", "account",
+    "evidence", "source", "record", "document", "memo", "report", "timeline",
+    "pattern", "symbolism", "conspiracy", "dark web", "onion", "leak",
+    "whistleblow", "history", "government", "archive", "corroborat",
+    "analysis", "investigat", "provenance", "citation", "fact", "truth",
+    "compare", "cross-reference", "cross reference", "research",
+]
+OFF_DOMAIN_HINTS = [
+    "how tall is", "recipe for", "weather in", "write a poem", "joke",
+    "horoscope", "what is your favorite", "stock tip", "cook", "play a game",
+    "dating advice", "what should i wear", "math homework:",
+]
+
+# --- PII redaction (dark-web/OSINT safety; never store what we don't need) ---
+PII_PATTERNS = [
+    (r"[\w.+-]+@[\w-]+\.[\w.-]+", "<email>"),
+    (r"\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b", "<phone>"),
+    (r"\b\d{3}-\d{2}-\d{4}\b", "<ssn>"),
+    (r"\b(?:\d[ -]*?){13,19}\b", "<card>"),
+]
+
+
+class GuardResult:
+    __slots__ = ("ok", "flags", "redacted")
+
+    def __init__(self, ok, flags, redacted):
+        self.ok = ok
+        self.flags = flags
+        self.redacted = redacted
+
+    def __repr__(self):
+        return f"GuardResult(ok={self.ok}, flags={self.flags})"
+
+
+def redact(text):
+    out = text
+    for pat, sub in PII_PATTERNS:
+        out = re.sub(pat, sub, out)
+    return out
+
+
+def check_input(text, domain_hints=ON_DOMAIN_HINTS,
+                off_hints=OFF_DOMAIN_HINTS):
+    """Flag unsafe/off-topic input; return redacted text + flags.
+
+    ok=False means the loop should ask the user to rephrase (never silently
+    drop — human-in-the-loop)."""
+    low = text.lower()
+    flags = []
+    for pat, name in INJECTION_PATTERNS:
+        if re.search(pat, low):
+            flags.append(f"injection:{name}")
+    if flags:
+        return GuardResult(False, flags, redact(text))
+    hit = sum(1 for h in domain_hints if h in low)
+    off = sum(1 for h in off_hints if h in low)
+    if off > hit:
+        flags.append("off-topic")
+        return GuardResult(False, flags, redact(text))
+    return GuardResult(True, flags, redact(text))
+
+
+def check_output(decision, require_verdict=True):
+    """Validate a decision dict before it reaches the user (output validation).
+
+    Rules: a verdict must exist; a non-abstain verdict must carry a confidence;
+    the abstain policy must be honored (abstained decisions say so)."""
+    flags = []
+    verdict = (decision.get("verdict") or "").strip()
+    conf = (decision.get("confidence") or "").strip()
+    if require_verdict and not verdict:
+        flags.append("missing-verdict")
+    if verdict and verdict.lower() != "not enough information" and not conf:
+        flags.append("missing-confidence")
+    if decision.get("abstained") and verdict.lower() != "not enough information":
+        flags.append("abstain-mismatch")
+    if not decision.get("abstained") and (decision.get("p") or 0.0) < 0.0:
+        flags.append("negative-probability")
+    return GuardResult(not flags, flags, "")

research/helix.py

@@ -0,0 +1,145 @@
+"""DNA-helix-style persistent memory for the analyst.
+
+Two complementary strands are stored per case:
+  - the CLAIM strand (what was asserted)
+  - the EVIDENCE strand (what the record said)
+The "rungs" are the values/entities that link them (years, percents, counts,
+times, names). When a new case arrives, recall() walks the strands for rung
+overlap and can answer from memory before re-reasoning -- the "remember what
+it needs to remember, when it needs to remember it" loop.
+
+Memory is append-only and deduplicated by rung signature. Saved as JSONL so it
+persists across sessions (closed loop: analyze -> write -> recall).
+"""
+import json
+import re
+import threading
+import time
+import uuid
+from pathlib import Path
+
+lock = threading.Lock()
+
+
+def rungs(text):
+    """Extract the linking values: numbers, years, times, percentages."""
+    out = []
+    out += re.findall(r"\b(?:19|20)\d{2}\b", text)
+    out += re.findall(r"\b\d{1,2}:\d{2}\b", text)
+    out += re.findall(r"\b\d+(?:,\d{3})*\.?\d*%?\b", text)
+    return sorted(set(out))
+
+
+def normalize(v):
+    v = v.strip().lower()
+    return v.replace("not enough information", "not_enough_info")
+
+
+class HelixMemory:
+    def __init__(self, path="data/helix_memory.jsonl"):
+        self.path = Path(path)
+        self.records = []
+        self._load()
+
+    def _load(self):
+        if not self.path.exists():
+            return
+        for line in self.path.open(encoding="utf-8"):
+            line = line.strip()
+            if line:
+                try:
+                    self.records.append(json.loads(line))
+                except Exception:
+                    pass
+
+    def write(self, claim, evidence, verdict, confidence, reasoning, agreed=True, mind=None,
+              case_id="default", source_ids=None, tags=None, salience=1.0,
+              privacy="case-local"):
+        sig = (mind,) + tuple(rungs(claim + " " + evidence)) if mind else tuple(rungs(claim + " " + evidence))
+        for r in self.records:
+            if tuple(r.get("sig", [])) == sig:
+                return r  # already remembered
+        rec = {
+            "id": "mem-" + uuid.uuid4().hex[:12],
+            "claim": claim, "evidence": evidence, "verdict": verdict,
+            "confidence": confidence, "reasoning": reasoning,
+            "agreed": bool(agreed), "sig": list(sig), "mind": mind,
+            "case_id": case_id, "source_ids": list(source_ids or []),
+            "tags": sorted(set(tags or [])), "salience": float(salience),
+            "privacy": privacy, "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        }
+        with lock:
+            self.records.append(rec)
+            with self.path.open("a", encoding="utf-8") as f:
+                f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+        return rec
+
+    @staticmethod
+    def _words(text):
+        return set(re.findall(r"[a-z]{4,}", text.lower()))
+
+    @staticmethod
+    def _jaccard(a, b):
+        if not a or not b:
+            return 0.0
+        return len(a & b) / len(a | b)
+
+    def recall(self, claim, evidence=""):
+        """Return prior record ONLY for a true repeat (same values, same claim)."""
+        matches = self.recall_many(claim, evidence, limit=1)
+        return matches[0] if matches else None
+
+    def recall_many(self, claim, evidence="", limit=5, case_id=None, tags=None):
+        """Rank related memories by rungs, words, tags, salience, and case scope."""
+        text = claim + " " + evidence
+        new_rungs, new_words = set(rungs(text)), self._words(text)
+        if not new_rungs and not new_words:
+            return []
+        wanted = set(tags or [])
+        scored = []
+        for record in self.records:
+            if case_id is not None and record.get("case_id", "default") != case_id:
+                continue
+            old_words = self._words(record.get("claim", "") + " " + record.get("evidence", ""))
+            old_rungs = set(record.get("sig", []))
+            rung_score = len(new_rungs & old_rungs) / max(1, len(new_rungs | old_rungs))
+            word_score = self._jaccard(new_words, old_words)
+            tag_score = len(wanted & set(record.get("tags", []))) / max(1, len(wanted)) if wanted else 0.0
+            score = 0.55 * rung_score + 0.35 * word_score + 0.10 * tag_score
+            if score > 0.05:
+                scored.append((score * max(0.1, float(record.get("salience", 1.0))), record))
+        scored.sort(key=lambda pair: pair[0], reverse=True)
+        return [{**record, "recall_score": round(score, 4)} for score, record in scored[:limit]]
+
+    def bridges(self, claim, evidence="", tags=None, limit=5):
+        """Cross case/domain recall: never require the same case to bridge."""
+        return self.recall_many(claim, evidence, limit=limit, tags=tags)
+
+    def forget(self, record_id=None, claim=None):
+        """User-controlled deletion; rewrites the JSONL atomically."""
+        before = len(self.records)
+        self.records = [r for r in self.records if not ((record_id and r.get("id") == record_id) or (claim and r.get("claim") == claim))]
+        removed = before - len(self.records)
+        if removed:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = self.path.with_suffix(self.path.suffix + ".tmp")
+            with tmp.open("w", encoding="utf-8") as fh:
+                for record in self.records:
+                    fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+            tmp.replace(self.path)
+        return removed
+
+    def consolidate(self):
+        """Deduplicate exact signatures while preserving the highest-salience record."""
+        best = {}
+        for record in self.records:
+            key = tuple(record.get("sig", []))
+            if key not in best or record.get("salience", 1.0) > best[key].get("salience", 1.0):
+                best[key] = record
+        self.records = list(best.values())
+        return self.stats()
+
+    def stats(self):
+        return {"records": len(self.records), "cases": len(set(r.get("case_id", "default") for r in self.records)),
+                "source_backed": sum(bool(r.get("source_ids")) for r in self.records),
+                "bridges": sum(bool(r.get("tags")) for r in self.records)}

research/index.py

@@ -0,0 +1,82 @@
+"""Tiny on-device retrieval index (TF-IDF, numpy) over corpus/raw texts."""
+
+import argparse
+import math
+import re
+import time
+from collections import Counter
+from pathlib import Path
+
+import numpy as np
+
+STOP = set("the a an of to in on for and or is are was were be been has have had it its this that with as at by from".split())
+
+
+def tokens(text: str):
+    return [w for w in re.findall(r"[a-z0-9']+", text.lower()) if w not in STOP and len(w) > 2]
+
+
+class TinyIndex:
+    def __init__(self):
+        self.docs = []          # list of (key, path, text)
+        self.terms = {}
+        self.df = Counter()     # doc frequencies
+        self.tfidf = None
+
+    def add(self, key, path, text):
+        self.docs.append((key, path, text))
+        for t in set(tokens(text)):
+            self.df[t] += 1
+
+    def build(self):
+        vocab = {t for t in self.df}
+        self.terms = {t: i for i, t in enumerate(sorted(vocab))}
+        n = len(self.docs)
+        rows, cols, vals = [], [], []
+        for di, (_, _, text) in enumerate(self.docs):
+            for t, c in Counter(tokens(text)).items():
+                if t in self.terms:
+                    idf = math.log((n + 1) / (self.df[t] + 1)) + 1
+                    rows.append(di); cols.append(self.terms[t]); vals.append(c * idf)
+        m = np.zeros((n, len(self.terms)), dtype=np.float32)
+        m[rows, cols] = vals
+        norms = np.linalg.norm(m, axis=1, keepdims=True)
+        norms[norms == 0] = 1
+        self.tfidf = m / norms
+
+    def query(self, q: str, k: int = 5):
+        if self.tfidf is None:
+            self.build()
+        v = np.zeros(len(self.terms), dtype=np.float32)
+        for t, c in Counter(tokens(q)).items():
+            if t in self.terms:
+                v[self.terms[t]] = c * (math.log((len(self.docs) + 1) / (self.df[t] + 1)) + 1)
+        if v.sum() == 0:
+            return []
+        v = v / np.linalg.norm(v)
+        scores = self.tfidf @ v
+        order = np.argsort(-scores)[:k]
+        return [(self.docs[i][0], float(scores[i])) for i in order if scores[i] > 0]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--corpus", default="corpus/raw")
+    ap.add_argument("--query", default=None)
+    ap.add_argument("--k", type=int, default=5)
+    args = ap.parse_args()
+
+    idx = TinyIndex()
+    for f in sorted(Path(args.corpus).glob("*.txt")):
+        idx.add(f.stem, str(f), f.read_text(encoding="utf-8", errors="ignore"))
+    print(f"indexed {len(idx.docs)} docs", flush=True)
+
+    if args.query:
+        for key, score in idx.query(args.query, args.k):
+            print(f"{score:.3f} {key} (corpus/raw/{key}.txt)", flush=True)
+    else:
+        print("usage: research/index.py --query 'query text' [--k 5]")
+
+
+if __name__ == "__main__":
+    main()

research/journalism.py

@@ -0,0 +1,194 @@
+"""Journalism suite facade — one call, the whole desk (tiny-model-journalism).
+
+suite_report(name, docs, claims) runs every deterministic forensic layer over
+a case and returns a single markdown notebook, saved as a durable CaseFile:
+  provenance ledger + chain-of-custody
+  timeline + gap/cliff/anachronism detection
+  framing / language forensics
+  cross-domain pattern synthesis
+  entity relationship graph
+  pre-publication adversarial review
+
+No model inference anywhere in this file — the suite materializes; the brain
+reasons over the surfaced leads.
+
+Usage:
+    from research.journalism import suite_report
+    md = suite_report("bridge_case", docs, claims)
+"""
+import hashlib
+import re
+from pathlib import Path
+
+from research.provenance import ProvenanceLedger, evaluate_source_policy
+from research.timeline import TimelineAnalyzer
+from research.framing import FramingAnalyzer
+from research.patterns import CrossDomainPatterns
+from research.entitygraph import EntityGraph
+from research.editorial_review import editorial_review
+from research.casefile import CaseFile
+
+_EVENT_LINE = re.compile(r"^(?:NOTE|EVENT|OPEN):\s*(.+)$")
+
+
+def suite_report(name, docs, claims=(), events=(), topics=()):
+    """Run the full forensic desk and fail closed on uncorroborated claims.
+
+    A non-empty claim verdict enters the CaseFile only when its source bundle
+    meets SOP 09: two independent usable sources with one strong source, each
+    carrying URL, retrieval time, content hash, and triage metadata. Otherwise
+    the claim remains an unresolved lead with its missing evidence recorded.
+    """
+    cf = CaseFile(name)
+    led = ProvenanceLedger(path=None)
+    tl = TimelineAnalyzer()
+    fr = FramingAnalyzer()
+    pat = CrossDomainPatterns()
+    eg = EntityGraph()
+    source_inputs = {}
+
+    for d in docs:
+        sid = d.get("source_id") or d.get("title") or "doc"
+        tier = d.get("tier", "unverified")
+        text = d.get("text", "")
+        retrieved_at = d.get("retrieved_at", d.get("retrieved", ""))
+        content_sha256 = d.get("content_sha256") or hashlib.sha256(
+            text.encode("utf-8", errors="replace")).hexdigest()
+        triage = d.get("triage") if isinstance(d.get("triage"), dict) else {}
+        led.register_source(sid, d.get("title", sid), tier=tier,
+                            url=d.get("url", ""), date=d.get("date", ""),
+                            retrievable=d.get("retrievable", True),
+                            independent=d.get("independent", True),
+                            origin=d.get("origin", ""),
+                            content_sha256=content_sha256, triage=triage,
+                            retrieved=retrieved_at)
+        source_inputs[sid] = {
+            "source_id": sid,
+            "url": d.get("url", ""),
+            "origin": d.get("origin", ""),
+            "retrieved_at": retrieved_at,
+            "content_sha256": content_sha256,
+            "independent": d.get("independent", True),
+            "retrievable": d.get("retrievable", True),
+            "triage": triage,
+        }
+        fr.add_doc(sid, text)
+        eg.add_doc(sid, text)
+        pat.add_strand(d.get("domain") or sid, text)
+        if d.get("date"):
+            tl.add_event(d["date"], d.get("title", sid), sid)
+        cf.add_source(sid, d.get("title", sid), tier=tier,
+                      url=d.get("url", ""), date=d.get("date", ""),
+                      retrievable=d.get("retrievable", True),
+                      independent=d.get("independent", True),
+                      origin=d.get("origin", ""), content_sha256=content_sha256,
+                      triage=triage, retrieved_at=retrieved_at)
+
+    for e in events:
+        tl.add_event(e["when"], e["what"], e.get("source_id", "-"))
+        cf.add_event(e["when"], e["what"], e.get("source_id", "-"))
+
+    policy_results = []
+    for c in claims:
+        sids = c.get("source_ids", [])
+        policy = evaluate_source_policy([source_inputs[sid] for sid in sids
+                                         if sid in source_inputs])
+        requested_verdict = c.get("verdict", "")
+        verdict, confidence = requested_verdict, c.get("confidence", "")
+        missing = c.get("missing", "")
+        if requested_verdict and not policy["verified"]:
+            verdict, confidence = "not enough information", "LOW"
+            policy_missing = "source policy: " + policy["reason"]
+            missing = "; ".join(part for part in (missing, policy_missing) if part)
+        policy_results.append({"claim": c["claim"], "policy": policy,
+                               "effective_verdict": verdict})
+        led.record_claim(c["claim"], sids, verdict=verdict, confidence=confidence)
+        cf.add_claim(c["claim"], sids, verdict=verdict, confidence=confidence,
+                     source_policy=policy)
+        if verdict:
+            cf.add_finding("main", c["claim"], verdict, confidence, sids,
+                           missing=missing, source_policy=policy)
+
+    md = [
+        f"# Journalism Suite: {name}",
+        f"documents: {len(docs)} | claims: {len(claims)} | "
+        f"events: {len(events)}",
+        "",
+        "---", "",
+        led.report(), "", "---", "",
+        tl.report(), "", "---", "",
+        fr.report(topics), "", "---", "",
+        pat.report(), "", "---", "",
+        eg.report(), "", "---", "",
+        _source_policy_section(policy_results), "", "---", "",
+        _review_section(claims, led), "",
+        "---", "",
+        "## CaseFile",
+        f"saved: {cf.path}",
+        "",
+        cf.export_markdown(),
+    ]
+    return "\n".join(md)
+
+
+def _source_policy_section(results):
+    lines = ["# Source Policy Gate", ""]
+    if not results:
+        lines.append("- no claim verdicts to gate")
+        return "\n".join(lines)
+    for result in results:
+        policy = result["policy"]
+        label = "VERIFIED SOURCE POLICY" if policy["verified"] else "LEAD ONLY"
+        lines.append(f"- [{label}] {result['claim']} -> {result['effective_verdict']} "
+                     f"({policy['reason']}; independent usable: "
+                     f"{policy['independent_usable']}, strong: "
+                     f"{policy['independent_strong']})")
+    return "\n".join(lines)
+
+
+def _review_section(claims, ledger):
+    lines = ["# Pre-Publication Adversarial Review", ""]
+    if not claims:
+        lines.append("- no claims to review")
+        return "\n".join(lines)
+    for c in claims:
+        sids = c.get("source_ids", [])
+        indep = [s for s in ledger.sources.values()
+                 if s.source_id in sids and s.independent and s.retrievable]
+        r = editorial_review(c["claim"], sources=len(indep),
+                             counter_evidence=c.get("counter_evidence", False),
+                             has_dates=c.get("has_dates", False))
+        lines.append(f"## {r['claim']}")
+        lines.append(f"**{r['summary']}** ({r['flags']} flags)")
+        for card in r["cards"]:
+            lines.append(f"- [{card['status']}] {card['item']}: "
+                         f"{card['detail']}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def docs_from_library(library_dir, max_chars=12000):
+    """Load library docs into the facade's doc schema (deterministic)."""
+    out = []
+    for p in sorted(Path(library_dir).glob("*")):
+        if not p.is_file():
+            continue
+        text = p.read_text(encoding="utf-8", errors="replace")[:max_chars]
+        out.append({"source_id": p.name, "title": p.stem, "text": text,
+                    "tier": "unverified", "url": "", "date": "",
+                    "domain": "library", "independent": True,
+                    "retrievable": True})
+    return out
+
+
+def claims_from_ledger(lines):
+    """Parse case ledger lines into claims (NOTE:/VERDICT:/OPEN: prefixes)."""
+    claims = []
+    for line in lines:
+        m = _EVENT_LINE.match(line.strip())
+        if m:
+            body = m.group(1).strip()
+            claims.append({"claim": body[:160], "source_ids": [],
+                           "verdict": "", "confidence": "",
+                           "counter_evidence": False, "has_dates": False})
+    return claims

research/orchestrator.py

@@ -0,0 +1,211 @@
+"""Parallel research orchestrator for the tiny researcher (suit layer).
+
+The 25M brain cannot spawn agents by itself. This module is the on-device
+equivalent: one shared model instance (a single analyst brain) with N worker
+threads, each running the SOP agent loop under a different research angle.
+Network and dark-web retrieval run in parallel; model inference is serialized
+by a lock (one brain, many hands), so concurrent torch forward passes never
+race on the 8-core device.
+
+Pipeline:
+  planner    -> deterministic angle split of the task
+  workers    -> N x agent.run_case (parallel I/O, locked inference)
+  synthesis  -> merge findings, dedup sources, flag cross-agent conflicts
+                (consensus + contradiction report), final analyst verdict
+
+Usage:
+  .venv/bin/python research/orchestrator.py --case "Verify: ..." --agents 4 \\
+      --sop dark_web_research --ckpt ckpt/tiny25m_sft_f
+"""
+
+import argparse
+import json
+import re
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import torch
+
+from model.config import TinyLiquidConfig
+from model.utils import latest_ckpt
+from model.tiny_liquid import TinyLiquid
+from data.tokenizer import load_tokenizer
+from research.agent import load_sop, run_case, list_sops
+from research.room import build_index
+from research.structured import analyst_report
+
+ROOT = Path(__file__).resolve().parents[1]
+
+ANGLES = {
+    "core": ("CORE CLAIM — verify the central claim itself against the record "
+             "first; cite the deciding document."),
+    "provenance": ("SOURCES & PROVENANCE — hunt the primary records behind the "
+                   "claim: who created them, when, and whether the provenance "
+                   "is verifiable."),
+    "timeline": ("TIMELINE & SEQUENCE — reconstruct the order of events and "
+                 "dates; flag gaps and after-the-fact records."),
+    "contradiction": ("CONTRADICTION & DISCREPANCY — find records that conflict "
+                      "with the claim or with each other; quantify the conflict."),
+    "pattern": ("PATTERN & CROSS-DOMAIN — look for recurring motifs, unusual "
+                "clusters, or links across domains the other angles would miss."),
+}
+ORDER = ["core", "provenance", "timeline", "contradiction", "pattern"]
+
+VERDICT_RE = re.compile(r"Verdict:\s*([^.\n]+)\.", re.IGNORECASE)
+
+# Opposite verdict pairs: agents landing on both sides of one of these is a
+# real cross-agent conflict worth surfacing in the merged report.
+OPPOSITES = {
+    "true": "false", "false": "true",
+    "contradiction": "not a contradiction", "not a contradiction": "contradiction",
+    "refutes": "supports", "supports": "refutes",
+    "overclaim": "understates", "understates": "overclaim",
+}
+
+
+def angles_for(task: str, n: int = 4) -> list[str]:
+    """Deterministic angle split: core claim always first, then the lenses that
+    match the task domain, capped at n."""
+    low = task.lower()
+    wanted = ["core"]
+    for name in ORDER[1:]:
+        if n <= len(wanted):
+            break
+        wanted.append(name)
+    return wanted[:max(1, min(n, len(ANGLES)))]
+
+
+def _verdicts(ledger):
+    return [m.group(1).strip().lower() for e in ledger
+            for m in [VERDICT_RE.search(e)] if m]
+
+
+def _worker(model, tok, task, idx, sop_text, angle, lock, max_steps):
+    try:
+        plan, ledger = run_case(model, tok, task, idx, sop_text,
+                                max_steps=max_steps, lock=lock, angle=ANGLES[angle])
+        notes = [e[5:].strip() for e in ledger if e.startswith("NOTE:")]
+        return {"angle": angle, "ok": True, "steps": len(plan), "plan": plan,
+                "ledger": ledger, "notes": notes, "verdicts": _verdicts(ledger)}
+    except Exception as e:  # one bad angle must not kill the swarm
+        return {"angle": angle, "ok": False, "error": str(e)[:300],
+                "steps": 0, "plan": [], "ledger": [], "notes": [], "verdicts": []}
+
+
+def run_parallel(model, tok, task, idx, sop_text, n=4, lock=None,
+                 max_steps=5, library="data/library"):
+    """Spawn n worker agents under distinct angles. I/O runs in parallel;
+    model inference is serialized by `lock` (one shared brain)."""
+    angles = angles_for(task, n)
+    lock = lock or threading.Lock()
+    results = []
+    with ThreadPoolExecutor(max_workers=len(angles)) as pool:
+        futs = {pool.submit(_worker, model, tok, task, build_index(library),
+                            sop_text, a, lock, max_steps): a for a in angles}
+        for fut in as_completed(futs):
+            results.append(fut.result())
+    results.sort(key=lambda r: ORDER.index(r["angle"]) if r["angle"] in ORDER else 99)
+    return results
+
+
+def _conflicts(results):
+    """Pairs of opposite verdicts reached by different agents."""
+    pairs = []
+    seen = set()
+    for i, a in enumerate(results):
+        for j, b in enumerate(results):
+            if i >= j:
+                continue
+            for va in a.get("verdicts", []):
+                for vb in b.get("verdicts", []):
+                    if OPPOSITES.get(va) == vb or OPPOSITES.get(vb) == va:
+                        key = tuple(sorted((a["angle"], b["angle"], va, vb)))
+                        if key not in seen:
+                            seen.add(key)
+                            pairs.append({"agents": [a["angle"], b["angle"]],
+                                          "verdicts": [va, vb]})
+    return pairs
+
+
+def synthesize(task, results, idx):
+    """Merge the swarm: grouped findings, deduped sources, conflicts, summary."""
+    ok = [r for r in results if r["ok"]]
+    findings = []
+    for r in ok:
+        for note in r["notes"]:
+            findings.append({"angle": r["angle"], "note": note})
+    sources = [{"key": k, "path": str(p)} for k, p, _ in idx.docs]
+    conflicts = _conflicts(ok)
+    completed = len(ok)
+    summary = (
+        f"{len(findings)} findings across {completed}/{len(results)} agents "
+        f"({', '.join(r['angle'] for r in ok) or 'none'}); "
+        f"{len(sources)} documents in the library; "
+        f"{len(conflicts)} cross-agent conflict(s) flagged."
+    )
+    return {
+        "task": task,
+        "agents_total": len(results),
+        "agents_ok": completed,
+        "angles": [r["angle"] for r in ok],
+        "findings": findings,
+        "sources": sources[:40],
+        "conflicts": conflicts,
+        "summary": summary,
+        "analyst": None,
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--case", default=None)
+    ap.add_argument("--agents", type=int, default=4, help="parallel agents (default 4)")
+    ap.add_argument("--sop", default=None, help="procedure stem, e.g. dark_web_research")
+    ap.add_argument("--list-sops", action="store_true")
+    ap.add_argument("--ckpt", default="ckpt/distill")
+    ap.add_argument("--tok", default="data/tokenizer.json")
+    ap.add_argument("--library", default="data/library")
+    ap.add_argument("--max-new", type=int, default=200)
+    ap.add_argument("--max-steps", type=int, default=5)
+    ap.add_argument("--threads", type=int, default=8)
+    args = ap.parse_args()
+
+    if args.list_sops:
+        list_sops()
+        return
+
+    task = args.case or sys.stdin.read().strip()
+    assert task, "no case provided (--case or stdin)"
+
+    torch.set_num_threads(args.threads)
+    tok = load_tokenizer(args.tok)
+    ckpt = latest_ckpt(args.ckpt)
+    assert ckpt, f"no checkpoints in {args.ckpt}"
+    sd = torch.load(ckpt, map_location="cpu")
+    cfg = TinyLiquidConfig(vocab_size=tok.get_vocab_size(),
+                           **{k: v for k, v in sd["config"].items() if k != "vocab_size"})
+    model = TinyLiquid(cfg)
+    model.load_state_dict(sd["model"])
+    model.eval()
+    print(f"loaded {ckpt} (step {sd.get('step', '?')})", flush=True)
+
+    sop_text = load_sop(args.sop, task)
+    idx = build_index(args.library)
+    print(f"swarm: {args.agents} agents | library docs: {len(idx.docs)}", flush=True)
+
+    results = run_parallel(model, tok, task, idx, sop_text,
+                           n=args.agents, max_steps=args.max_steps,
+                           library=args.library)
+    merged = synthesize(task, results, build_index(args.library))
+    merged["analyst"] = analyst_report(model, tok, task, persona_id=1,
+                                       max_scratch=args.max_new // 2,
+                                       max_reason=args.max_new // 4)
+
+    print("\n=== MERGED REPORT ===")
+    print(json.dumps(merged, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

research/patterns.py

@@ -0,0 +1,105 @@
+"""Cross-domain pattern synthesis over memory strands (journalism suite 4).
+
+The owner's closed-loop insight: every domain sits in one system, so a rung
+(number, year, name) or theme repeated across UNRELATED domains is a lead.
+The suite finds the overlap; the human decides whether the connection is
+causal, coincidental, or symbolic. Cards carry the base-rate caveat so a
+repeated number is never auto-promoted to a conclusion.
+
+Basis: helix rung model (research/helix.py) + memory skill's cross-domain
+reinforcement doctrine (tiny-model-memory).
+
+Usage:
+    from research.patterns import CrossDomainPatterns
+    p = CrossDomainPatterns()
+    p.add_strand("economics", "the 1929 crash... gold standard...")
+    p.add_strand("religion", "Genesis... serpent... 1929...")
+    p.report()
+"""
+import re
+from collections import defaultdict
+
+from research.helix import rungs
+
+THEMES = [
+    "serpent", "snake", "eye", "pyramid", "coin", "flood", "plague", "fire",
+    "tower", "gate", "seal", "crown", "star", "dove", "wolf", "mirror",
+    "key", "blood", "gold", "iron", "wall", "circle", "garden", "beast",
+    "mark", "number", "trumpet", "scroll", "angel", "dragon",
+]
+_NAME = re.compile(r"\b[A-Z][a-z]{2,20}(?:\s+[A-Z][a-z]{2,20}){0,2}\b")
+
+
+class CrossDomainPatterns:
+    def __init__(self):
+        self.strands = []       # list of {"domain", "text"}
+
+    def add_strand(self, domain, text):
+        self.strands.append({"domain": domain, "text": text})
+
+    def domains(self):
+        return sorted({s["domain"] for s in self.strands})
+
+    def shared_rungs(self):
+        """Rungs (numbers/years/times/names) present in >=2 different domains."""
+        by_rung = defaultdict(dict)
+        for s in self.strands:
+            vals = set(rungs(s["text"]))
+            for v in vals:
+                by_rung[v][s["domain"]] = by_rung[v].get(s["domain"], 0) + 1
+        out = []
+        for v, doms in by_rung.items():
+            if len(doms) >= 2:
+                out.append({"rung": v, "domains": sorted(doms),
+                            "strength": min(doms.values())})
+        return sorted(out, key=lambda c: -c["strength"])
+
+    def theme_overlap(self):
+        """Themes present in >=2 different domains."""
+        by_theme = defaultdict(set)
+        for s in self.strands:
+            low = s["text"].lower()
+            for t in THEMES:
+                if t in low:
+                    by_theme[t].add(s["domain"])
+        return [{"theme": t, "domains": sorted(d)}
+                for t, d in by_theme.items() if len(d) >= 2]
+
+    def names(self, min_domains=2):
+        """Proper-noun co-occurrence across domains (loose entity bridge)."""
+        by_name = defaultdict(set)
+        for s in self.strands:
+            for m in _NAME.finditer(s["text"]):
+                by_name[m.group(0)].add(s["domain"])
+        return [{"name": n, "domains": sorted(d)}
+                for n, d in by_name.items() if len(d) >= min_domains]
+
+    def report(self):
+        lines = ["# Cross-Domain Pattern Synthesis", ""]
+        lines.append(f"domains: {', '.join(self.domains())}")
+        lines.append("")
+        lines.append("## Shared rungs (numbers/years/times)")
+        sr = self.shared_rungs()
+        for c in sr[:20]:
+            lines.append(f"- `{c['rung']}` (strength {c['strength']}) appears in "
+                         f"{', '.join(c['domains'])}")
+            lines.append("  - LEAD: check whether causal, coincidental, or symbolic")
+        if not sr:
+            lines.append("- no cross-domain rungs")
+        lines.append("")
+        lines.append("## Theme overlap")
+        for c in self.theme_overlap()[:20]:
+            lines.append(f"- '{c['theme']}' in {', '.join(c['domains'])}")
+            lines.append("  - LEAD: base-rate check first; repeated themes are "
+                         "common in text")
+        if not self.theme_overlap():
+            lines.append("- no cross-domain themes")
+        lines.append("")
+        lines.append("## Name bridges")
+        for c in self.names()[:20]:
+            lines.append(f"- '{c['name']}' in {', '.join(c['domains'])}")
+        if not self.names():
+            lines.append("- no cross-domain name bridges")
+        lines.append("")
+        lines.append("_Every card above is a LEAD, never a verdict._")
+        return "\n".join(lines)

research/probe.py

@@ -0,0 +1,62 @@
+"""Run a fixed set of forensic probes through a TinyLiquid checkpoint.
+
+Usage:
+  .venv/bin/python research/probe.py --ckpt ckpt/distill
+"""
+
+import argparse
+from pathlib import Path
+
+import torch
+
+from model.config import TinyLiquidConfig, CONFIGS
+from model.tiny_liquid import TinyLiquid
+from model.utils import latest_ckpt
+from data.tokenizer import load_tokenizer
+
+PROBES = [
+    ("analyst", "Find discrepancies between: Account A: The meeting started at 9am and ended at 11am. Account B: The meeting started at 9am and ran until noon."),
+    ("analyst", "Two accounts describe the same event. Account A: 'No officials were present.' Account B: 'An official arrived later.' What can you conclude?"),
+    ("analyst", "Evaluate this claim: 'Crime in the city doubled last year because of the new policy.' The report shows incidents rose from 1,000 to 2,000 while reporting methods changed."),
+    ("analyst", "What are the weak links in a theory claiming one actor caused three unrelated disasters?"),
+    ("skeptic", "Attack this conclusion: 'The stock dropped after the announcement, so investors rejected the announcement.'"),
+]
+
+
+def parse_args():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--ckpt", default="ckpt/distill")
+    ap.add_argument("--tok", default="data/tokenizer.json")
+    ap.add_argument("--max-new", type=int, default=140)
+    ap.add_argument("--threads", type=int, default=8)
+    return ap.parse_args()
+
+
+def main():
+    args = parse_args()
+    torch.set_num_threads(args.threads)
+    tok = load_tokenizer(args.tok)
+    ckpt = latest_ckpt(args.ckpt)
+    assert ckpt, f"no checkpoints in {args.ckpt}"
+    sd = torch.load(ckpt, map_location="cpu")
+    cfg_dict = dict(sd.get("config", CONFIGS["tiny10m"]))
+    cfg = TinyLiquidConfig(vocab_size=tok.get_vocab_size(),
+                           **{k: v for k, v in cfg_dict.items() if k != "vocab_size"})
+    model = TinyLiquid(cfg)
+    model.load_state_dict(sd["model"])
+    model.eval()
+    print(f"== {ckpt} (step {sd.get('step','?')}) ==\n", flush=True)
+
+    P_TOKEN = {"analyst": "<|analyst|>", "skeptic": "<|skeptic|>"}
+    P_ID = {"analyst": 1, "skeptic": 2}
+    for persona, prompt in PROBES:
+        p = P_TOKEN[persona] + "<|user|>" + prompt + "<|assistant|>"
+        ids = tok.encode(p).ids
+        out = model.generate(tok, ids, persona_id=P_ID[persona], max_new=args.max_new,
+                             temperature=0.65, top_k=40, repetition_penalty=1.4,
+                             no_repeat_ngram_size=4)
+        print(f"--- [{persona}] {prompt}\n{tok.decode(out[len(ids):])}\n", flush=True)
+
+
+if __name__ == "__main__":
+    main()

research/procedures_research.md

@@ -0,0 +1,161 @@
+# How Codex-style agents run per-task procedures — and how we give TinyLiquid the same power
+
+Date: 2026-07-31
+Sources: official Codex manual (cached copy, `/tmp/openai-docs-cache/codex-manual.md`)
+         -> "Custom instructions with AGENTS.md", "Best practices", "Prompting", "Plan mode".
+
+## 1. What the "little task bar" actually is
+
+The task bar you see in Codex/GPT agents is not a single feature. It is three
+mechanisms working together:
+
+1. **A plan / task list (the visible bar).** For multi-step work the agent
+   maintains an explicit list of steps and updates status as it goes
+   (plan mode, `update_plan`-style item updates). The list is *external state*
+   — it lives in the loop, not in the model's weights — so the model never has
+   to remember where it is.
+2. **Durable procedures (the invisible rules).** Repo instructions in
+   `AGENTS.md` (and overrides) are *loaded into the prompt at start* and stay
+   in context. They encode: repo layout, how to run/test, conventions,
+   constraints, do-not rules, and "what done means". Multiple `AGENTS.md` files
+   are layered global -> project -> subdirectory; closer files override, and
+   the whole chain is capped at 32 KiB.
+3. **A tool loop with guardrails.** The agent proposes tool calls (shell,
+   search, file edits), the harness executes them under a sandbox, feeds
+   results back, and the agent re-plans. Approval rules gate destructive
+   actions. This loop is what makes a small per-step model behave like a
+   careful operator instead of a chatbox.
+
+Official manual facts used here:
+- "Codex reads `AGENTS.md` files before doing any work... Discovery follows
+  this precedence order: global scope, then project scope (walking from
+  project root down to cwd), merged root-to-leaf; files closer to the current
+  directory override earlier guidance."
+- "A good `AGENTS.md` covers: repo layout..., build/test/lint commands,
+  engineering conventions..., constraints and do-not rules, what done means
+  and how to verify work."
+- Prompt best practice: give the agent **Goal / Context / Constraints /
+  Done-when** so it stays scoped.
+- Guidance is deliberately *short and practical*: "start with the basics,
+  then add new rules only after you notice repeated mistakes."
+
+## 2. The general pattern (model-agnostic)
+
+Any agent (tiny or huge) gets powerful from this 4-layer stack:
+
+    Layer 0  Weights         language skill learned at pretraining
+    Layer 1  Prompt          goal + context + constraints + done-when
+    Layer 2  Procedures      durable SOP text injected per task (AGENTS.md analog)
+    Layer 3  Loop            plan list + tool calls + result feedback + stop rules
+
+Big models rely on Layer 1-3 being *understandable*. Tiny models fail there:
+they lose track, drift off procedure, and cannot hold long context. So for a
+tiny model we must make Layers 2-3 *external and mechanical*:
+
+- procedures are files we choose and inject (never rely on memory);
+- the loop, not the model, tracks step state (ledger, remaining steps);
+- outputs that matter are produced with constrained decoding (verdict,
+  confidence) so the model cannot emit an off-protocol final answer;
+- and we *train* the model to obey the procedure format (SFT + DPO on
+  procedure-following examples), so Layer 2 becomes learned behavior, not just
+  a prompt trick.
+
+## 3. What we already have in this repo
+
+- `research/analyst.py`  — dual-mind: analyst pass + skeptic attack pass.
+- `research/structured.py` — constrained verdict/confidence decoding (the
+  "cannot emit off-protocol" guarantee).
+- `research/room.py` — interactive environment: model issues RETRIEVE/READ/
+  NOTE/VERDICT actions against a local TF-IDF library; ledger = working memory
+  that exceeds the model's parameters.
+- `research/crawl.py` + `research/index.py` — clearnet/Tor fetch and local
+  search (the "library" and "dark web" tools).
+- Training: SFT examples already use `<|scratchpad|> ... <|final|>` so the
+  model is trained to think-then-answer in protocol form.
+
+Gap vs. the Codex pattern: procedures are currently *one hard-coded SOP in
+analyst.py*, not a per-task library; there is no "plan" step list; there is no
+procedure-aware training data. This task closes exactly that gap.
+
+## 4. Design: the SOP layer for TinyLiquid
+
+We mirror the Codex stack 1:1, adapted to 7.8M params:
+
+    Codex                          TinyLiquid equivalent
+    ---------------------------    -----------------------------------------
+    AGENTS.md files                research/sop_library/*.md (per task)
+    plan/task list                 research/agent.py ledger + step list
+    tool loop                      RETRIEVE/READ/NOTE/VERDICT + crawl/index
+    guardrails                     stop rules in each SOP + read-only shell
+    trained behavior               sft_sop.jsonl + prefs_sop.jsonl (SFT+DPO)
+
+### 4.1 SOP library (procedures = AGENTS.md analog)
+
+`research/sop_library/00_common.md` — universal rules every procedure obeys:
+evidence over assertion, name missing evidence, primary-source checks, two
+independent sources per factual claim, confidence on every verdict,
+"cannot confirm" beats speculation, never a final verdict — decision support.
+
+Task procedures (all authorized-research/OSINT, never illegal action):
+- `claim_verification.md`       — decompose -> source -> corroborate -> date ->
+                                  provenance -> verdict.
+- `cross_source_discrepancy.md` — align two accounts, list deltas, classify
+                                  each delta (typo/ambiguity/conflict), find
+                                  which source changes the story.
+- `pattern_finding.md`          — collect events, cluster, look for common
+                                  cause/escalation, test against null
+                                  hypothesis, state pattern strength.
+- `timeline_reconstruction.md`  — anchor to dated primary records, gap list,
+                                  contradiction list, don't fill gaps with
+                                  inference.
+- `historical_truth.md`         — compare past reporting to later records,
+                                  identify what was hidden/late/corrected.
+- `politics_analysis.md`        — separate interests from evidence, track
+                                  provenance of talking points, rate
+                                  spin vs fact.
+- `dark_web_research.md`        — authorized OSINT; use crawler + Tor proxy
+                                  for .onion; rate-limit; never purchase,
+                                  never access CSAM/credential dumps, never
+                                  engage; document chain of custody.
+- `terminal_control.md`         — read-only first, dry-run, log every command,
+                                  no destructive ops without explicit approval,
+                                  kill long-running jobs, verify outputs.
+- `source_triage.md`            — score sources on independence, recency,
+                                  proximity to primary record, track record.
+
+### 4.2 Trained behavior (baking procedures into weights)
+
+1. `data/gen_sop_sft.py` -> `data/sft_sop.jsonl`
+   Task prompts that name a procedure; the assistant answer opens a scratchpad
+   that applies the procedure's steps to the material, then a `<|final|>` with
+   verdict + confidence. This teaches: (a) read the procedure, (b) follow it
+   stepwise, (c) never skip to a conclusion.
+2. `data/gen_sop_sft.py` also emits `data/prefs_sop.jsonl` (DPO pairs):
+   chosen = answer that follows the SOP; rejected = confident answer that
+   skipped the procedure. Teaches the *preference*: protocol beats fluency.
+3. Train order after current NLP retrain:
+   `forensic SFT -> distill SFT -> SOP SFT (mixed with distill set) -> DPO -> code stage`.
+
+### 4.3 Inference loop (`research/agent.py`)
+
+- `--sop <name>` injects the procedure text into the user turn (Layer 2).
+- The model works the case with RETRIEVE/READ/NOTE actions; agent.py enforces
+  a max-step plan (Layer 3), the ledger is external memory.
+- At the end the loop runs the structured decoder (verdict + confidence),
+  then a skeptic pass, then emits a JSON report that audits which SOP steps
+  were actually completed and which gaps remain.
+- This is the "task bar": the user sees the step ledger update, exactly like
+  watching Codex work through its plan.
+
+## 5. Best path forward (current situation)
+
+- NLP retrain (`ckpt/nlp`, ~7.8M params) is running on-device and producing
+  coherent text (val_loss ~3.13, best yet).
+- Next: finish NLP, then SFT on `sft_distill_mix` (forensic + teacher
+  distillation), then SOP SFT + DPO, then probe, then the code stage, then
+  GGUF quantization for edge deployment.
+- Rule of thumb that big-tech recommends for tiny models: *more
+  high-quality, narrowly-scoped examples beats raw scale*. The 114 gold
+  distillation examples plus the new procedure-conditioned sets are the right
+  shape; we keep each SFT set focused and mix them at ~1:1 with the base
+  forensic set to avoid forgetting.

research/provenance.py

@@ -0,0 +1,272 @@
+"""Source credibility + provenance ledger (journalism suite layer 1).
+
+The tiny head never grades a source. The SUITE keeps the ledger: every source
+registered with a tier, a retrieval date, a retrievability flag, and an
+independence mark; every claim recorded with its full chain-of-custody. A
+credibility score is a deterministic heuristic (tier weight x retrievability
+x independence), never a model opinion.
+
+Basis: Bellingcat OSINT chain-of-custody / evidence standards; repo suit
+decision (2026-08-07) to carry provenance tiers on every verdict record.
+
+Usage:
+    from research.provenance import ProvenanceLedger
+    led = ProvenanceLedger()
+    led.register_source("s1", title="DOT filing", tier="verified-leak",
+                        url="file://dot_2010.txt", date="2010-06-01")
+    led.record_claim("bridge opened 2010", ["s1", "s2"])
+    led.report()
+"""
+import json
+import re
+import time
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+
+TIERS = ("verified-leak", "secondary", "unverified", "claim")
+TIER_WEIGHT = {"verified-leak": 1.0, "secondary": 0.6, "unverified": 0.3, "claim": 0.1}
+TRIAGE_FIELDS = ("independence", "proximity", "recency", "track", "interest")
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+def evaluate_source_policy(sources):
+    """Apply SOP 09 deterministically to a retrieved evidence bundle.
+
+    A source may be strong (13-15), usable only with corroboration (9-12), or
+    a lead (<9). A claim is eligible for a verified result only when it has two
+    independent usable sources with distinct origins, including one strong
+    source. Metadata is mandatory so a later reviewer can reproduce the trail.
+
+    ``sources`` is a sequence of mappings with ``source_id``, ``url``,
+    ``retrieved_at`` (or ``retrieved``), ``content_sha256``, ``independent``,
+    ``retrievable``, and five 0-3 ``triage`` scores. The scores are supplied by
+    the retrieval/review workflow, never inferred by the language model.
+    """
+    cards = []
+    independent_origins = {}
+    for raw in sources or ():
+        source = raw if isinstance(raw, dict) else {}
+        triage = source.get("triage") if isinstance(source.get("triage"), dict) else {}
+        errors = []
+        source_id = str(source.get("source_id", "")).strip()
+        url = str(source.get("url", "")).strip()
+        retrieved_at = str(source.get("retrieved_at") or source.get("retrieved") or "").strip()
+        digest = str(source.get("content_sha256", "")).lower().strip()
+        if not source_id:
+            errors.append("missing-source-id")
+        if not url:
+            errors.append("missing-url")
+        if not retrieved_at:
+            errors.append("missing-retrieved-at")
+        if not _SHA256.fullmatch(digest):
+            errors.append("missing-or-invalid-content-sha256")
+
+        values = {}
+        for field in TRIAGE_FIELDS:
+            value = triage.get(field)
+            if type(value) is not int or not 0 <= value <= 3:
+                errors.append(f"invalid-triage-{field}")
+            else:
+                values[field] = value
+        score = sum(values.values()) if len(values) == len(TRIAGE_FIELDS) else 0
+        retrievable = bool(source.get("retrievable", False))
+        independent = bool(source.get("independent", False))
+        valid = not errors
+        usable = valid and retrievable and score >= 9
+        strong = usable and score >= 13
+        classification = "strong" if strong else "usable" if usable else "lead"
+        origin = str(source.get("origin") or url).strip()
+        card = {
+            "source_id": source_id,
+            "origin": origin,
+            "score": score,
+            "classification": classification,
+            "usable": usable,
+            "strong": strong,
+            "independent": independent,
+            "errors": errors,
+        }
+        cards.append(card)
+        if usable and independent:
+            previous = independent_origins.get(origin)
+            if previous is None or card["score"] > previous["score"]:
+                independent_origins[origin] = card
+
+    independent = list(independent_origins.values())
+    strong = [card for card in independent if card["strong"]]
+    verified = len(independent) >= 2 and bool(strong)
+    if verified:
+        reason = "two-independent-usable-sources-including-one-strong"
+    elif len(independent) < 2:
+        reason = "fewer-than-two-independent-usable-sources"
+    else:
+        reason = "no-strong-source"
+    return {
+        "verified": verified,
+        "reason": reason,
+        "independent_usable": len(independent),
+        "independent_strong": len(strong),
+        "sources": cards,
+    }
+
+
+@dataclass
+class SourceRecord:
+    source_id: str
+    title: str
+    tier: str = "unverified"
+    url: str = ""
+    date: str = ""
+    retrieved: str = ""
+    retrievable: bool = True
+    independent: bool = True   # not a re-publication of another recorded source
+    notes: str = ""
+    origin: str = ""
+    content_sha256: str = ""
+    triage: dict = field(default_factory=dict)
+
+    def credibility(self) -> float:
+        w = TIER_WEIGHT.get(self.tier, 0.1)
+        score = w * (1.0 if self.retrievable else 0.4)
+        if not self.independent:
+            score *= 0.5
+        return round(score, 3)
+
+
+@dataclass
+class ClaimRecord:
+    claim: str
+    source_ids: list = field(default_factory=list)
+    verdict: str = ""
+    confidence: str = ""
+    noted: str = ""
+
+    def chain(self, ledger: "ProvenanceLedger") -> list:
+        out = []
+        for sid in self.source_ids:
+            s = ledger.sources.get(sid)
+            if s:
+                out.append({"source_id": sid, "tier": s.tier,
+                            "title": s.title, "url": s.url,
+                            "credibility": s.credibility(),
+                            "retrievable": s.retrievable,
+                            "independent": s.independent})
+        return out
+
+
+class ProvenanceLedger:
+    """Deterministic source + claim ledger with chain-of-custody reports."""
+
+    def __init__(self, path="data/casefiles/provenance.json"):
+        self.path = Path(path) if path else None
+        self.sources = {}
+        self.claims = []
+        self._load()
+
+    def _load(self):
+        if not self.path or not self.path.exists():
+            return
+        try:
+            d = json.loads(self.path.read_text(encoding="utf-8"))
+            for s in d.get("sources", []):
+                rec = SourceRecord(**{k: v for k, v in s.items()
+                                      if k in SourceRecord.__dataclass_fields__})
+                self.sources[rec.source_id] = rec
+            self.claims = [ClaimRecord(**{k: v for k, v in c.items()
+                                          if k in ClaimRecord.__dataclass_fields__})
+                           for c in d.get("claims", [])]
+        except (ValueError, TypeError):
+            pass
+
+    def _save(self):
+        if not self.path:
+            return
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps({
+            "sources": [asdict(s) for s in self.sources.values()],
+            "claims": [asdict(c) for c in self.claims],
+        }, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    def register_source(self, source_id, title, tier="unverified", url="",
+                        date="", retrievable=True, independent=True, notes="",
+                        origin="", content_sha256="", triage=None, retrieved=""):
+        if tier not in TIERS:
+            raise ValueError(f"tier must be one of {TIERS}")
+        if source_id in self.sources:
+            rec = self.sources[source_id]
+            rec.title = title
+            rec.tier = tier
+            rec.url = url or rec.url
+            rec.date = date or rec.date
+            rec.retrieved = retrieved or rec.retrieved
+            rec.retrievable = retrievable
+            rec.independent = independent
+            rec.notes = notes or rec.notes
+            rec.origin = origin or rec.origin
+            rec.content_sha256 = content_sha256 or rec.content_sha256
+            rec.triage = triage if triage is not None else rec.triage
+            return rec
+        rec = SourceRecord(source_id=source_id, title=title, tier=tier, url=url,
+                           date=date, retrieved=retrieved or time.strftime("%Y-%m-%d"),
+                           retrievable=retrievable, independent=independent,
+                           notes=notes, origin=origin,
+                           content_sha256=content_sha256, triage=triage or {})
+        self.sources[source_id] = rec
+        self._save()
+        return rec
+
+    def record_claim(self, claim, source_ids, verdict="", confidence=""):
+        rec = ClaimRecord(claim=claim, source_ids=list(source_ids),
+                          verdict=verdict, confidence=confidence,
+                          noted=time.strftime("%Y-%m-%d"))
+        self.claims.append(rec)
+        self._save()
+        return rec
+
+    def corroboration(self, claim_text):
+        """Independent sources behind a claim (dedup by url)."""
+        urls = set()
+        out = []
+        for c in self.claims:
+            if c.claim.strip().lower() != claim_text.strip().lower():
+                continue
+            for s in c.chain(self):
+                if s["independent"] and s["retrievable"] and s["url"] not in urls:
+                    urls.add(s["url"])
+                    out.append(s)
+        return out
+
+    def single_source(self):
+        """Claims backed by at most one independent source."""
+        return [c for c in self.claims
+                if sum(1 for s in c.chain(self) if s["independent"]) <= 1]
+
+    def unverified(self):
+        return [s for s in self.sources.values() if s.tier in ("unverified", "claim")]
+
+    def report(self):
+        lines = ["# Provenance Ledger", ""]
+        lines.append("## Sources")
+        for s in self.sources.values():
+            lines.append(f"- `{s.source_id}` [{s.tier}] {s.title} "
+                         f"(cred {s.credibility():.2f}, "
+                         f"{'retrievable' if s.retrievable else 'NOT retrievable'}, "
+                         f"{'independent' if s.independent else 'derived'})")
+        lines.append("")
+        lines.append("## Claims & chain-of-custody")
+        for c in self.claims:
+            chain = c.chain(self)
+            lines.append(f"- **{c.claim}**")
+            for s in chain:
+                lines.append(f"  - {s['source_id']} ({s['tier']}, cred {s['credibility']:.2f})")
+            if not chain:
+                lines.append("  - UNSUBSTANTIATED: no recorded source")
+        lines.append("")
+        lines.append("## Flags")
+        if self.single_source():
+            lines.append(f"- single-source claims: {len(self.single_source())}")
+        if self.unverified():
+            lines.append(f"- unverified/claim-tier sources: {len(self.unverified())}")
+        if not self.claims:
+            lines.append("- no claims recorded")
+        return "\n".join(lines)

research/researcher_model_survey.md

@@ -0,0 +1,91 @@
+# Researcher/Truth-Verifier Tiny Model — Multi-Site Research Survey
+
+Sources pulled Aug 2026: arXiv (TinyStories, Self-Consistency, LoRA, FEVER,
+Toolformer, Let's Verify Step by Step, Chain-of-Verification, DeepSeek-R1, Phi-1,
+LIMA, Small-LM Survey) + HuggingFace (SmolLM, LoRA) + measured experiments on THIS
+tablet (MoE, tower, scan numerics, corpus mixing). Each item: source, what it
+means, verdict for this project.
+
+## PROVEN — adopt
+1. Domain-constrained base (TinyStories, 2305.07759): tiny models speak coherently
+   only inside a simple, constrained domain. We did this (val 2.47, coherent).
+2. Curriculum pretraining (SmolLM, HF blog): easy -> hard data ordering. Adopted:
+   balanced shuffled corpus (train_phase2b.bin).
+3. Verifiable 3-way claim verdicts (FEVER, 1803.05355): labels checkable from the
+   prompt alone -> learnable at 16M. Core task, Stage A.
+4. Few hundred handcrafted examples shape style (LIMA, 2305.11206): our kd skill.
+5. Textbook-quality curation beats scale (Phi-1, 2306.11644): our kd skill.
+6. RL on verifiable rewards elicits reasoning (DeepSeek-R1, 2501.12948): Stage E —
+   reward = probe-verdict correctness, rule-checkable.
+7. Process supervision > outcome (2305.20050): reward scratchpad STEPS, not only
+   the final verdict. Stage D design.
+8. Self-verification cuts hallucination (Chain-of-Verification, 2309.11495):
+   draft -> verify -> revise SOP in every example.
+9. Tool loops are learnable (Toolformer, 2302.04761): model emits search/retrieve
+   actions; the CLIENT executes them under Tor.
+10. Self-consistency decoding (2203.11171): sample N reasoning paths, majority
+    vote. Zero training cost — implement in the client at inference.
+11. LoRA (2106.09685): parameter-efficient SFT. Use for fast SFT iterations on the
+    frozen trunk; full SFT only when we commit a final stage.
+12. Anti-imitation guardrail (False-Promise): never train soft logits from a big
+    model; verifiable targets only.
+
+## EXPERIMENTAL — pilot, don't bet the pipeline
+- Persona-routed sparse experts (dual-mind in one forward): measured partial
+  specialization (corr 0.435). Pilot: explicit persona->router bias, target corr < 0.3.
+- Process-rewarded SFT: train on scratchpad step sequences, not just final verdicts.
+- Abstention/calibration: measure calibration on abstain rows (silent record ->
+  must say unsubstantiated). Tune confidence thresholds by probe score.
+- Long context via liquid recurrence: test seq 512-1024 on this device; recurrence
+  may extend effective context cheaply (our arch's natural advantage).
+- RAG lite: BM25 index over the user's document folder; client retrieves, model
+  analyzes. No training change.
+
+## UNPROVEN / REJECTED (documented decisions, do not re-run)
+- 250-tiny-expert MoE: router collapse (51/250 used), no per-step win. Needs a
+  load-balance loss + shared-base/LoRA experts before any retry.
+- Width upscaling 320->512: val 6.1-7.7 vs baseline 2.58. Rejected.
+- Pure logit distillation from a big model: evidence says imitation degrades
+  small students. Rejected for skills.
+- Helix/DNA "memory" as architecture magic: treat as CLIENT-side memory/state, not
+  a model-level capability. No training-time promise.
+- Symbolic/neural hybrids: untested, high complexity, no evidence at 16M.
+
+## OUR OWN DESIGN (the unique moat — not in any other model we found)
+- Source-DNA provenance tags: every claim tagged primary/secondary/anonymous,
+  independent-origin count, hash/PGP/corroborated status. Trained output class.
+- Gap / "blotchy" detector: explicitly trained ABSENCE-finding (missing actor,
+  date, period, attachment, named source). Abstains on whatever rests on the gap.
+- Symbolism decoder with base-rate discipline: exact textual patterns score higher
+  than motif association; always correlation-tagged, never proof.
+- Told-vs-not-told timeline: dated grid of assertions vs records; gaps and
+  ordering anomalies are the output.
+- Thread-tracer: "does X connect to Y?" broken into hops, each hop must be a real
+  record; proven / unproven / broken.
+- SOP-conditioned scratchpad: procedure baked into the FORMAT (source -> evidence
+  -> self-check -> verdict -> missing), so the model follows it by conditioning,
+  not by remembering.
+- Safe dark-web research SOP: verify .onion against trusted mirror, PGP check,
+  no JS/downloads/logins, no identity. In the client + trained dialogue.
+- Mistake-driven closed loop: probe -> triage individual mistakes -> handcraft
+  targeted gold -> retrain -> re-probe. The pipeline moat (grants story).
+- Confidence + abstention discipline: calibrated by construction; abstaining is a
+  correct answer, never a failure.
+
+## What I'm adopting now (priority order)
+1. Finish continue-pretrain (running; ETA ~9h).
+2. Stage A: grow FEVER-style verdict gold to ~500+ rows (handcrafted).
+3. Stage B: discrepancy/pattern gold (v12) growth.
+4. Stage C: symbolism + gap gold (v13) growth — the unique niche features.
+5. Stage D: self-verify draft->revise samples.
+6. Stage E: DPO/preference with rule-checkable verdict reward.
+7. Inference (client): self-consistency majority vote + BM25 doc retrieval +
+   provenance-tag tool. GGUF/int8 export.
+8. Guardrails: no soft-imitation; window-shuffle corpora; scan chunk 16;
+   load-balance loss if MoE is ever revisited.
+
+## Community niche (release framing)
+- First open tiny model explicitly for TRUTH-VERIFYING RESEARCH with trained
+  discrepancy/pattern/gap/symbolism analysis + safe dark-web SOP — a defined
+  niche no other small model we found occupies. Release as GGUF + Q8, with the
+  probe scorecard and an honest README (capabilities + limitations + safety).

research/rlvr.py

@@ -0,0 +1,58 @@
+"""RLVR reward harness — verifiable reward on the deterministic spine.
+
+The policy is the tiny analyst head; the reward is RULES, never the head
+grading itself (DeepSeek-R1: reasoning emerges when reward is rule-checkable;
+here it is OUR verify spine / probe labels):
+  +1   policy verdict == gold verdict (exact, constrained vocabulary)
+   0   honest abstention ("not enough information" / "unsubstantiated")
+  -1   policy verdict contradicts the gold
+  +0.2 citation present AND its value appears in the evidence record
+  -0.2 citation present but the value is NOT in the record (fabricated anchor)
+Self-reported confidence is NEVER rewarded (anti-calibrated, measured).
+
+Usage:
+    from research.rlvr import reward, reward_card
+    r = reward(gold="refutes", policy="refutes", citation="1982",
+               evidence="the deed file states 1982")
+"""
+import re
+import time
+
+ABSTAIN = {"not enough information", "unsubstantiated", "abstain",
+           "cannot provide", "no record"}
+_VALUE = re.compile(r"\d{1,2}:\d{2}\b|\b(?:19|20)\d{2}\b|"
+                    r"\b\d+(?:,\d{3})*\.?\d*%?\b")
+
+
+def _cited_in_evidence(citation, evidence):
+    ev = evidence.lower()
+    for v in _VALUE.findall(citation):
+        if v in ev:
+            return True
+    return False
+
+
+def reward(gold, policy, citation="", evidence=""):
+    """Rule reward for one (probe, policy) step. Deterministic."""
+    pv = policy.strip().lower()
+    gv = gold.strip().lower()
+    if pv in ABSTAIN:
+        verdict = 0.0
+    elif pv == gv:
+        verdict = 1.0
+    else:
+        verdict = -1.0
+    cit = 0.0
+    if citation:
+        cit = 0.2 if _cited_in_evidence(citation, evidence) else -0.2
+    return {"verdict": verdict, "citation": cit,
+            "total": round(verdict + cit, 3)}
+
+
+def reward_card(gold, policy, citation="", evidence="", probe=""):
+    """Full chain-of-custody trace for RLVR logs (auditable)."""
+    r = reward(gold, policy, citation, evidence)
+    return {"probe": probe, "gold": gold, "policy": policy,
+            "citation": citation, "evidence": evidence[:160],
+            "verdict": r["verdict"], "citation_reward": r["citation"],
+            "total": r["total"], "ts": time.strftime("%Y-%m-%d %H:%M")}

research/room.py

@@ -0,0 +1,152 @@
+"""The Analyst Room: an interactive environment for the tiny model.
+
+The model does not answer from memory alone. It works a case:
+  RETRIEVE <query>   -> ask the library for relevant passages
+  READ <key>         -> open a document
+  NOTE <text>        -> write a finding to the evidence ledger
+  VERDICT            -> produce the constrained final report
+
+Every action is appended to the ledger, and the ledger is re-read on the
+next step, so the model accumulates a case across many turns (working
+memory that exceeds its parameter count).
+
+Usage:
+  .venv/bin/python research/room.py --ckpt ckpt/distill
+  .venv/bin/python research/room.py --ckpt ckpt/distill --case "Was the 2019 outage caused by the truck seen nearby?"
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+from model.config import TinyLiquidConfig, CONFIGS
+from model.tiny_liquid import TinyLiquid
+from model.utils import latest_ckpt
+from data.tokenizer import load_tokenizer
+from research.index import TinyIndex
+from research.structured import analyst_report, _decode_phrase
+
+ACTIONS = ["RETRIEVE", "READ", "NOTE", "VERDICT"]
+MAX_STEPS = 6
+
+
+def load_model(args):
+    torch.set_num_threads(args.threads)
+    tok = load_tokenizer(args.tok)
+    ckpt = latest_ckpt(args.ckpt)
+    sd = torch.load(ckpt, map_location="cpu")
+    cfg = TinyLiquidConfig(vocab_size=tok.get_vocab_size(),
+                           **{k: v for k, v in sd["config"].items() if k != "vocab_size"})
+    model = TinyLiquid(cfg)
+    model.load_state_dict(sd["model"])
+    model.eval()
+    return tok, model, ckpt
+
+
+def build_index(library_dir: str, extra_dirs=("corpus/raw",)):
+    idx = TinyIndex()
+    for f in sorted(Path(library_dir).glob("*.txt")):
+        idx.add(f.stem, str(f), f.read_text(encoding="utf-8", errors="ignore"))
+    for d in extra_dirs:
+        for f in sorted(Path(d).glob("*.txt")):
+            idx.add(f"{Path(d).name}/{f.stem}", str(f),
+                    f.read_text(encoding="utf-8", errors="ignore"))
+    idx.build()
+    return idx
+
+
+def format_ledger(ledger):
+    return "\n".join(f"[{i+1}] {e}" for i, e in enumerate(ledger[-8:])) or "(empty)"
+
+
+def make_context(prompt, ledger, hits):
+    ctx = (
+        "You are working a research case. Use the case file and library hits. "
+        "Reply with exactly one line: ACTION: <RETRIEVE|READ|NOTE|VERDICT> then ARG: <text>\n"
+        f"CASE FILE:\n{format_ledger(ledger)}\n"
+    )
+    if hits:
+        ctx += "LIBRARY HITS:\n" + hits + "\n"
+    return ctx + f"TASK: {prompt}"
+
+
+def hit_text(idx, query, k=3, max_chars=500):
+    rows = idx.query(query, k)
+    parts = []
+    for key, score in rows:
+        text = idx.docs[[d[0] for d in idx.docs].index(key)][2] if key in [d[0] for d in idx.docs] else ""
+        parts.append(f"<{key} (score {score:.2f})> " + text[:max_chars].replace("\n", " "))
+    return "\n".join(parts)
+
+
+def read_doc(idx, key):
+    for k, path, text in idx.docs:
+        if k == key:
+            return text[:2000]
+    return "(document not found)"
+
+
+def run_case(model, tok, prompt, idx, max_steps=MAX_STEPS):
+    ledger = []
+    for step in range(max_steps):
+        hits = ""
+        if ledger:
+            last = ledger[-1]
+            if last.startswith("RETRIEVE:"):
+                hits = hit_text(idx, last.split(":", 1)[1].strip())
+        ctx = make_context(prompt, ledger, hits)
+        ids = tok.encode("<|analyst|><|user|>" + ctx + "<|assistant|>ACTION:").ids
+        pre = len(ids)
+        ids = _decode_phrase(model, tok, ids, 1, ACTIONS)
+        action = tok.decode(ids[pre:]).strip().upper()
+        if action not in ACTIONS:
+            action = "NOTE"
+        # free-form argument
+        ids = ids + tok.encode(" ARG:").ids
+        arg = tok.decode(model.generate(tok, ids, persona_id=1, max_new=80,
+                                        temperature=0.5, top_k=40,
+                                        repetition_penalty=1.5, no_repeat_ngram_size=4)[len(ids):]).strip()
+        if action == "VERDICT" or action == "RETRIEVE" and step == max_steps - 1:
+            if action == "VERDICT" or step == max_steps - 1:
+                break
+        ledger.append(f"{action}: {arg}")
+        print(f"  [{step+1}] {action}: {arg}", flush=True)
+    return ledger
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--ckpt", default="ckpt/distill")
+    ap.add_argument("--tok", default="data/tokenizer.json")
+    ap.add_argument("--library", default="data/library")
+    ap.add_argument("--case", default=None)
+    ap.add_argument("--threads", type=int, default=8)
+    args = ap.parse_args()
+
+    tok, model, ckpt = load_model(args)
+    idx = build_index(args.library)
+    print(f"room open: {ckpt} | library docs: {len(idx.docs)}", flush=True)
+
+    if args.case:
+        ledger = run_case(model, tok, args.case, idx)
+        report = analyst_report(model, tok, args.case)
+        print("\n=== FINAL REPORT ===")
+        print(json.dumps({**report, "steps": ledger}, indent=2, ensure_ascii=False))
+        return
+
+    print("Analyst Room REPL (type a case or question; Ctrl-D to exit)")
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        ledger = run_case(model, tok, line, idx)
+        report = analyst_report(model, tok, line)
+        print("\n=== FINAL REPORT ===")
+        print(json.dumps({**report, "steps": ledger}, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

research/sop_library/00_common.md

@@ -0,0 +1,19 @@
+# SOP 00 — COMMON RULES (every procedure obeys these)
+
+PURPOSE: baseline truth-seeking rules. Never override them.
+
+1. Evidence over assertion. Every conclusion names the evidence it rests on.
+2. Name what is missing. If a checkable fact is unverified, say so.
+3. Primary source first. Records, filings, official documents, original
+   statements. Aggregators and commentary are leads, not sources.
+4. Two independent sources per factual claim. Same-source repetition is not
+   corroboration.
+5. Confidence on every verdict: HIGH / MEDIUM / LOW / cannot assess.
+6. "Cannot confirm" beats speculation. Never fill a gap with inference.
+7. Flag overclaims: words like "always / never / proven / everyone" are
+   hypotheses until evidenced.
+8. Decision support only. This output is analysis, never a verdict.
+9. If a step cannot be completed, record the blocker and continue; do not
+   fake completion.
+10. Audit trail: every material fact is traceable to a named document or
+    retrieval action.

research/sop_library/claim_verification.md

@@ -0,0 +1,19 @@
+# SOP 01 — CLAIM VERIFICATION
+
+USE WHEN: a single factual claim or statement must be checked.
+
+1. DECOMPOSE: split the claim into separate checkable assertions. Never judge
+   a multi-part claim as one unit.
+2. SOURCE: find the primary source (records, filings, official documents,
+   original statements), not commentary.
+3. CORROBORATE: require two independent sources. Same-source repetition is
+   not corroboration.
+4. DATE: fix when each source was made and when it was verified; stale
+   sources cannot verify current claims.
+5. PROVENANCE: who originated the claim, who amplified it, and what they had
+   to gain.
+6. VERDICT: per assertion -> true / false / mostly true / partially true /
+   mixed / unsubstantiated / overclaim / unverifiable. Confidence per verdict.
+7. Stop when: every assertion has a verdict and a named source chain.
+
+OUTPUT: verdict list + source chain + confidence + remaining gaps.

research/sop_library/cross_source_discrepancy.md

@@ -0,0 +1,18 @@
+# SOP 02 — CROSS-SOURCE DISCREPANCY
+
+USE WHEN: two or more accounts of the same event disagree.
+
+1. ALIGN: list both accounts side by side (who, what, when, where, how).
+2. DELTA: list every difference — numbers, names, dates, sequences, causes.
+3. CLASSIFY each delta: typo / ambiguity / emphasis / contradiction /
+   incompatible (both cannot be true).
+4. ROOT: for each contradiction, ask which source changes the story and what
+   evidence would settle it.
+5. TIMING: note if accounts were produced before/after the event (recency
+   bias, memory effects, post-hoc spin).
+6. VERDICT: which elements are confirmed, which conflict, which unverifiable.
+   Confidence per element.
+7. Stop when: every delta is classified and each open conflict names the
+   evidence that would settle it.
+
+OUTPUT: aligned accounts + delta table + conflicts + resolution evidence.

research/sop_library/dark_web_research.md

@@ -0,0 +1,20 @@
+# SOP 07 — DARK WEB / DEEP WEB RESEARCH (AUTHORIZED OSINT ONLY)
+
+USE WHEN: searching clearnet + Tor (.onion) sources for documents, claims, or
+patterns. Authorized research and OSINT only. Nothing illegal, ever.
+
+1. SCOPE: write the research question and the categories you will and will
+   not touch. Never: purchases, credentials/CSAM, malware, direct engagement
+   with actors. Never bypass access controls.
+2. CRAWL: use research/crawl.py (clearnet) and TOR_PROXY for .onion.
+   Rate-limit every host; crawl only authorized targets.
+3. TRIAGE: score each hit with SOP 09 (source triage) before quoting it.
+4. CHAIN: record url, fetch time, hash, and snippet for every used source
+   (chain of custody).
+5. VERIFY: treat dark-web claims as unverified leads until SOP 01 passes with
+   independent sources.
+6. STOP RULES: any result that escalates toward illegal content ends the
+   session immediately and is reported as a blocker, never opened further.
+7. Output: evidence list with chain-of-custody + verified/unverified labels.
+
+OUTPUT: scope statement + evidence chain + triage scores + open blockers.

research/sop_library/historical_truth.md

@@ -0,0 +1,17 @@
+# SOP 05 — HISTORICAL TRUTH / PAST-NEWS AUDIT
+
+USE WHEN: comparing what was reported earlier against what later records show.
+
+1. RETRIEVE: collect the earlier reporting (what was said, when, by whom).
+2. AFTERMATH: collect later records (corrections, retractions, disclosures,
+   official findings).
+3. DELTA: list what changed between early reporting and later record.
+4. HIDDEN: identify what was absent early and appeared late; ask who knew
+   and when (provenance).
+5. CORRECTED vs CORRUPTED: separate honest corrections from systematic
+   suppression or error patterns.
+6. VERDICT per claim: confirmed / corrected / retracted / contradicted /
+   still open. Confidence each.
+7. Stop when: early reporting, later record, and every delta are on the table.
+
+OUTPUT: before/after table + hidden-items list + verdicts + open items.

research/sop_library/pattern_finding.md

@@ -0,0 +1,15 @@
+# SOP 03 — PATTERN FINDING
+
+USE WHEN: looking for recurring structure across events or claims.
+
+1. COLLECT: list every event/claim with date, actor, and source.
+2. CLUSTER: group by similarity (actor, method, target, timing, claim shape).
+3. COMMON CAUSE: for each cluster, propose the mechanism that links members.
+4. NULL TEST: state what pattern would look like if the mechanism were false;
+   look for counterexamples deliberately.
+5. STRENGTH: rate pattern as weak (coincidence not excluded) / moderate /
+   strong (counterexamples searched, mechanism evidenced).
+6. ESCALATION: note whether clusters grow, accelerate, or repeat cyclically.
+7. Stop when: clusters, mechanism, counterexamples, and strength are stated.
+
+OUTPUT: cluster table + mechanism + counterexamples + strength rating.

research/sop_library/politics_analysis.md

@@ -0,0 +1,15 @@
+# SOP 06 — POLITICS / SPIN ANALYSIS
+
+USE WHEN: analyzing political statements, talking points, or disputes.
+
+1. SEPARATE: extract factual assertions vs. interests and framing.
+2. PROVENANCE: trace each talking point to its originator and amplifier.
+3. INTEREST: state what each party gains from the claim being accepted.
+4. EVIDENCE: apply SOP 01 (claim verification) to each factual assertion.
+5. SPIN RATE: label each statement fact / partial / spin / false, with the
+   missing context that would change the label.
+6. BOTH-SIDES TEST: apply the same standard to every party; unequal scrutiny
+   is itself a discrepancy to report.
+7. Stop when: assertions, interests, provenance, and spin labels are explicit.
+
+OUTPUT: assertion table + interest map + spin labels + unequal-scrutiny notes.

research/sop_library/source_triage.md

@@ -0,0 +1,19 @@
+# SOP 09 — SOURCE TRIAGE
+
+USE WHEN: deciding how much weight a source deserves.
+
+Score 0-3 each:
+1. INDEPENDENCE: 3 = unrelated to the parties, 0 = the party itself or paid
+   by it (note: self-statements are primary evidence, not corroboration).
+2. PROXIMITY: 3 = primary record (original document, filing, raw data),
+   0 = retold commentary.
+3. RECENCY: 3 = produced for the period in question, 0 = long after, with
+   memory/spin risk.
+4. TRACK: 3 = correct on comparable past claims, 0 = repeatedly wrong.
+5. INTEREST: 3 = nothing to gain, 0 = material stake in the claim.
+
+WEIGHT: 13-15 strong, 9-12 usable with corroboration, <9 lead only.
+Never rest a verdict on a single <9 source. Never count one source twice
+(same parent outlet / same wire story = one source).
+
+OUTPUT: per-source scores + weight class + corroboration requirement.

research/sop_library/terminal_control.md

@@ -0,0 +1,18 @@
+# SOP 08 — TERMINAL CONTROL
+
+USE WHEN: using the shell as part of research.
+
+1. READ-ONLY FIRST: list/search/inspect before anything that writes.
+2. DRY-RUN: prefer commands that preview effects (--dry-run, --check,
+   --diff) over direct execution.
+3. LOG: every command and its output is appended to the session ledger.
+4. NO DESTRUCTION: no rm -rf, no overwrites, no network mutations without
+   explicit approval. Default to new files.
+5. BOUNDS: timeouts on every fetch; kill runaway jobs; never run unknown
+   downloaded code.
+6. VERIFY: after a command, check the artifact exists and is sane (size,
+   head, checksum) before building on it.
+7. Stop when: the question is answered or a blocker is recorded with the
+   exact command that produced it.
+
+OUTPUT: command log + artifact checks + blockers.

research/sop_library/timeline_reconstruction.md

@@ -0,0 +1,16 @@
+# SOP 04 — TIMELINE RECONSTRUCTION
+
+USE WHEN: reconstructing what happened and when.
+
+1. ANCHOR: collect dated primary records first (documents, logs, official
+   statements with dates).
+2. ORDER: sort anchors chronologically; leave gaps explicit.
+3. GAP LIST: every undated or missing period is listed as a gap. Do not fill
+   gaps with inference.
+4. CONTRADICT: flag entries that clash on dates/order; keep both, mark
+   conflict.
+5. INFERENCE: any non-anchored assertion is labeled "inferred" with its
+   confidence.
+6. Stop when: anchors, gaps, conflicts, and inferences are all labeled.
+
+OUTPUT: dated anchor list + gap list + conflict list + labeled inferences.

research/structured.py

@@ -0,0 +1,117 @@
+"""Structured SOP decoding for the analyst persona.
+
+The scratchpad is generated freely; the verdict and confidence sections are
+decoded greedily under a controlled vocabulary, so the model cannot emit an
+unstructured or off-protocol answer for the final judgment.
+
+Usage:
+  from research.structured import analyst_report
+  report = analyst_report(model, tok, doc_text, max_scratch=140)
+"""
+
+import torch
+
+VERDICTS = [
+    "true", "false", "mostly true", "partially true", "mixed",
+    "unsubstantiated", "unsupported", "overclaim", "misleading",
+    "inaccurate", "unverifiable", "cannot confirm",
+    "not a discrepancy", "conflict", "no meaningful pattern",
+    # canonical set (eval_labels.CANON) — the tiny head must be able to emit these
+    "refutes", "not enough information", "not a contradiction",
+    "contradiction", "low confidence", "abstain", "cannot provide",
+]
+CONFIDENCES = ["HIGH", "MEDIUM", "LOW", "cannot assess"]
+
+
+def _decode_phrase(model, tok, ids, persona_id, allowed_phrases, max_new=16, fallback=None):
+    """Constrained greedy decode over the allowed phrases (longest-prefix).
+
+    Returns (exact_phrase_token_ids, ok). Never emits partial/garbage tokens:
+    - if a full allowed phrase is matched, its EXACT tokens are returned,
+    - otherwise the caller falls back (default: explicit abstain text),
+      so the model can never present an off-protocol verdict.
+    """
+    targets = [tok.encode(p).ids for p in allowed_phrases]
+    targets = [t for t in targets if t]
+    if not targets:
+        return [], False
+    progress = [0] * len(targets)
+    seq = torch.tensor([ids], dtype=torch.long)
+    for _ in range(max_new):
+        window = seq[:, -model.cfg.max_seq_len:]
+        logits = model(window, persona_ids=torch.tensor([persona_id]) if persona_id else None)
+        logits = logits[:, -1, :]
+        allowed = {}
+        for i, t in enumerate(targets):
+            if progress[i] < len(t):
+                allowed.setdefault(t[progress[i]], i)
+        if not allowed:
+            break
+        allowed_ids = torch.tensor(list(allowed.keys()), dtype=torch.long)
+        mask = torch.full_like(logits, -float("inf"))
+        mask[:, allowed_ids] = logits[:, allowed_ids]
+        nxt = int(mask.argmax().item())
+        seq = torch.cat([seq, torch.tensor([[nxt]], dtype=torch.long)], dim=1)
+        for i, t in enumerate(targets):
+            if progress[i] < len(t) and t[progress[i]] == nxt:
+                progress[i] += 1
+                if progress[i] == len(t):
+                    return t, True  # exact full-phrase tokens
+        # if we advanced no target further, stop (no clean path)
+        if not any(progress[i] > 0 and progress[i] < len(targets[i]) for i in range(len(targets))):
+            break
+    return [], False
+
+
+@torch.no_grad()
+def analyst_report(model, tok, doc, persona_id=1, max_scratch=140, max_reason=80):
+    """Two-pass structured analysis: scratchpad -> verdict -> confidence -> reasoning.
+
+    The verdict and confidence are CONSTRAINT-DECODED with a hard fallback: if the
+    head does not cleanly emit an allowed phrase, we return an explicit abstention
+    ("not enough information" / "cannot assess") and flag decode_failed=True.
+    Garbage is never presented as a verdict.
+    """
+    persona_tok = "<|analyst|>"
+    prompt = persona_tok + "<|user|>" + doc + "<|assistant|>"
+    ids = tok.encode(prompt).ids
+
+    # free-form scratchpad
+    scratch = model.generate(tok, ids, persona_id=persona_id, max_new=max_scratch,
+                             temperature=0.6, top_k=40, repetition_penalty=1.4,
+                             no_repeat_ngram_size=4)
+    ids = scratch
+
+    # constrained verdict (hard fallback -> abstention)
+    vids, vok = _decode_phrase(model, tok, ids, persona_id, VERDICTS)
+    if vok:
+        verdict = tok.decode(vids).strip()
+    else:
+        verdict = "not enough information"
+        vids = tok.encode(verdict).ids
+    ids = ids + vids
+
+    # constrained confidence
+    ids = ids + tok.encode(" Confidence: ").ids
+    cids, cok = _decode_phrase(model, tok, ids, persona_id, CONFIDENCES)
+    if cok:
+        confidence = tok.decode(cids).strip()
+    else:
+        confidence = "cannot assess"
+        cids = tok.encode(confidence).ids
+    ids = ids + cids
+
+    # free-form reasoning
+    ids = ids + tok.encode(" Reasoning: ").ids
+    reason = model.generate(tok, ids, persona_id=persona_id, max_new=max_reason,
+                            temperature=0.6, top_k=40, repetition_penalty=1.4,
+                            no_repeat_ngram_size=4)
+    reasoning = tok.decode(reason[len(ids):]).strip()
+
+    return {
+        "scratchpad": tok.decode(scratch[len(tok.encode(prompt).ids):]).strip(),
+        "verdict": verdict,
+        "confidence": confidence,
+        "reasoning": reasoning,
+        "decode_failed": not (vok and cok),
+    }

research/timeline.py

@@ -0,0 +1,145 @@
+"""Timeline reconstruction + gap detection (journalism suite layer 2).
+
+A good investigation reads the ABSENCES as much as the events. This module
+sorts dated events, measures intervals, and surfaces:
+  - gaps: intervals that exceed a heuristic threshold (2x median, >= 1 year)
+  - cliffs: active years whose neighbors are active but themselves silent
+  - anachronisms: an event whose text cites a year different from its date
+  - density: per-year event counts (where did the reporting thin out?)
+
+Deterministic only — the model reasons over the surfaced gaps; the suite
+never invents the missing event.
+
+Usage:
+    from research.timeline import TimelineAnalyzer
+    tl = TimelineAnalyzer()
+    tl.add_event("2010-05-01", "bridge opens per DOT filing", "s1")
+    tl.report()
+"""
+import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+_YEAR = re.compile(r"\b(19|20)\d{2}\b")
+
+
+@dataclass
+class Event:
+    when: str          # ISO date YYYY-MM-DD or year YYYY
+    what: str
+    source_id: str = "-"
+
+    def date(self):
+        """Sortable key: YYYY -> YYYY-01-01; ISO kept as-is."""
+        if len(self.when) == 4 and self.when.isdigit():
+            return f"{self.when}-01-01"
+        return self.when
+
+
+class TimelineAnalyzer:
+    def __init__(self):
+        self.events = []
+
+    def add_event(self, when, what, source_id="-"):
+        self.events.append(Event(when=when, what=what, source_id=source_id))
+
+    def sorted(self):
+        return sorted(self.events, key=lambda e: e.date())
+
+    def years(self):
+        out = defaultdict(int)
+        for e in self.events:
+            out[e.date()[:4]] += 1
+        return dict(sorted(out.items()))
+
+    def _gaps_raw(self, gap_min_days=None):
+        """(start_date, end_date, days) for each interval above threshold."""
+        ev = self.sorted()
+        if len(ev) < 2:
+            return []
+        deltas = []
+        for a, b in zip(ev, ev[1:]):
+            try:
+                deltas.append((_days(b.date()) - _days(a.date()), a, b))
+            except ValueError:
+                continue
+        if not deltas:
+            return []
+        median = sorted(d for d, _, _ in deltas)[len(deltas) // 2]
+        floor = gap_min_days or max(365, 2 * median)
+        return [(a, b, d) for d, a, b in deltas if d > floor]
+
+    def gaps(self, gap_min_days=None):
+        """Gap cards: missing period + the bookend events + absent line."""
+        out = []
+        for a, b, days in self._gaps_raw(gap_min_days):
+            out.append({
+                "from": a.date(),
+                "to": b.date(),
+                "days": days,
+                "between": [a.what, b.what],
+                "absent": f"no recorded event between {a.date()} and {b.date()} "
+                          f"({days} days) - what happened there?",
+            })
+        return out
+
+    def cliffs(self):
+        """Years silent while both neighbors have events (missing period)."""
+        ys = self.years()
+        if len(ys) < 2:
+            return []
+        lo, hi = int(min(ys)), int(max(ys))
+        out = []
+        for y in range(lo, hi + 1):
+            yk = str(y)
+            if ys.get(yk, 0) == 0 and ys.get(str(y - 1), 0) and ys.get(str(y + 1), 0):
+                out.append({"year": yk, "before": str(y - 1), "after": str(y + 1),
+                            "absent": f"year {yk} is silent between active "
+                                      f"years {y - 1} and {y + 1}"})
+        return out
+
+    def anachronisms(self):
+        """Event text cites a year that differs from its own date year."""
+        out = []
+        for e in self.events:
+            cited = set(_YEAR.findall(e.what))
+            if cited and e.date()[:4] not in cited:
+                out.append({"when": e.date(), "what": e.what,
+                            "cited_years": sorted(cited),
+                            "flag": "cited year != event date year"})
+        return out
+
+    def report(self, gap_min_days=None):
+        lines = ["# Timeline", ""]
+        lines.append("| date | event | source |")
+        lines.append("|---|---|---|")
+        for e in self.sorted():
+            lines.append(f"| {e.date()} | {e.what} | {e.source_id} |")
+        lines.append("")
+        lines.append("## Density (events/year)")
+        for y, n in self.years().items():
+            lines.append(f"- {y}: {n}")
+        lines.append("")
+        lines.append("## Gaps (what is absent)")
+        gaps = self.gaps(gap_min_days)
+        for g in gaps:
+            lines.append(f"- {g['absent']}")
+            lines.append(f"  - between: {g['between'][0]}  |  {g['between'][1]}")
+        if not gaps:
+            lines.append("- no gaps above threshold")
+        lines.append("")
+        lines.append("## Cliffs & anachronisms")
+        for c in self.cliffs():
+            lines.append(f"- {c['absent']}")
+        for a in self.anachronisms():
+            lines.append(f"- `{a['when']}` {a['what']} -> cites {a['cited_years']} "
+                         f"({a['flag']})")
+        if not self.cliffs() and not self.anachronisms():
+            lines.append("- none")
+        return "\n".join(lines)
+
+
+def _days(iso):
+    from datetime import date
+    y, m, d = (int(x) for x in iso.split("-"))
+    return date(y, m, d).toordinal()

research/user_journal.py

@@ -0,0 +1,132 @@
+"""Per-user journalist's notebook (deterministic suit memory).
+
+The tiny model cannot rederive "who the user is and what they care about" from
+weights. So the suit keeps a small, auditable USER JOURNAL: focus topics, active
+investigation threads, remembered corrections, and a tone preset. The journal is
+injected as a short context prefix on each analysis turn, and is updated with a
+hashed-simple key + rationalized text / questions. This is memory in the suit,
+not in the brain; when the user returns days later, the model "still knows them"
+like a good journalist knows their subject.
+
+Usage:
+    from research.user_journal import UserJournal
+    j = UserJournal()            # loads ./data/user_journal.json (creates default)
+    ctx = j.context()            # compact prompt-prefix string
+    j.note_thread(text)          # remember this turn as an active thread
+    j.note_fact(fact)            # pin a fact/correction the user cares about
+"""
+import json
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+# Editorial tone presets -> the single line we hand the model.
+TONES = {
+    "spock": "Tone: strictly logical, evidence-first, concise; state what is "
+             "unsupported instead of guessing.",  # default
+    "journalist": "Tone: probe the question; separate asserted fact from "
+                  "speculation; ask what source the user already trusts.",
+    "coach": "Tone: explain briefly and support the user's own reasoning, "
+             "correcting only where evidence demands.",
+    "concise": "Tone: compact and direct, no filler.",
+}
+DEFAULT_TONE = "spock"
+MAX_THREADS = 12
+MAX_FACTS = 12
+
+
+_default = {
+    "handle": "guest",
+    "tone": DEFAULT_TONE,
+    "focus": [],
+    "threads": [],
+    "facts": [],
+    "notes": "",
+    "corrections": [],
+    "last_seen": "",
+}
+
+
+class UserJournal:
+    def __init__(self, path=None):
+        self.path = Path(path) if path else ROOT / "data" / "user_journal.json"
+        self.data = dict(_default)
+        if self.path.exists():
+            try:
+                import json
+                self.data.update(json.loads(self.path.read_text()))
+            except Exception:
+                pass
+
+    def save(self):
+        import json, time
+        self.data["last_seen"] = time.strftime("%Y-%m-%d %H:%M")
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(self.data, indent=2, ensure_ascii=False))
+
+    # ---- reads ----
+    def context(self):
+        d = self.data
+        lines = ["\nJOURNAL (about the user, journal's private notes):"]
+        lines.append("handle: " + str(d.get("handle", "guest")))
+        if d.get("tone"):
+            lines.append(TONES.get(d["tone"], TONES[DEFAULT_TONE]))
+        if d.get("threads"):
+            lines.append("active threads: " + "; ".join(t if isinstance(t, str) else t.get("title","") for t in d["threads"][-MAX_THREADS:]))
+        if d.get("facts"):
+            lines.append("notes: " + " | ".join(str(f)[:140] for f in d["facts"][-MAX_FACTS:]))
+        if d.get("corrections"):
+            lines.append("remembered corrections: " + " | ".join(c[:140] for c in d["corrections"][-6:]))
+        lines.append("These are private notes. Use them to be relevant to THIS user, "
+                     "but do not state them back verbatim.\n")
+        return "\n".join(lines)
+
+    # ---- writes (suit heuristics) ----
+    def set_handle(self, name):
+        self.data["handle"] = (name or "guest").strip()
+
+    def set_tone(self, preset):
+        if preset in TONES:
+            self.data["tone"] = preset
+
+    def note_thread(self, text):
+        title = " ".join((text or "").split()[:12])
+        if not title:
+            return
+        threads = [t for t in self.data.setdefault("threads", []) if not (isinstance(t,str) and t==title)]
+        threads.append(title)
+        self.data["threads"] = threads[-MAX_THREADS:]
+
+    def note_fact(self, fact):
+        fact = (fact or "").strip()
+        if not fact:
+            return
+        self.data.setdefault("facts", []).append(fact)
+        self.data["facts"] = self.data["facts"][-MAX_FACTS:]
+
+    def note_focus(self, terms):
+        for t in (terms or []):
+            t = str(t).strip()
+            if t and t not in self.data.setdefault("focus", []):
+                self.data["focus"].append(t)
+        self.data["focus"] = self.data["focus"][-16:]
+
+    def remember_correction(self, text):
+        # A safety-corpus: if the user explicitly corrects us, keep it short.
+        low = text.lower()
+        if any(k in low for k in ("you're wrong", "that's wrong", "no, ", "correction", "actually ")):
+            self.data.setdefault("corrections", []).append(text[:160])
+
+    def snapshot(self):
+        return dict(self.data)
+
+
+if __name__ == "__main__":
+    import sys
+    j = UserJournal()
+    print(j.context())
+    print("---")
+    j.note_thread("wanted to verify the 2022 repaint permit narrative")
+    j.note_fact("user cares about timeline provenance across agencies")
+    print("after write snapshot keys:", sorted(j.snapshot().keys()))

research/verify.py

@@ -0,0 +1,60 @@
+"""Deterministic claim-vs-evidence verifier for the numeric spine.
+
+A 7.8M model cannot reliably copy values, so the hard comparison is done by
+rules instead of generation. The model's job is confined to what it actually
+can do: identifying the claim and the evidence segments. This module:
+
+  1. extracts values from the claim side and the evidence side,
+  2. compares them deterministically,
+  3. returns a verdict that cannot be hallucinated:
+       supports / refutes / not enough information / unclear.
+
+For any case it cannot resolve (no clean numeric pair), it says so -- it never
+fabricates an answer.
+"""
+import re
+
+
+def _nums(text):
+    return re.findall(r"\b\d+(?:,\d{3})*\.?\d*%?\b", text)
+
+
+def _years(text):
+    return re.findall(r"\b(?:19|20)\d{2}\b", text)
+
+
+def _times(text):
+    return re.findall(r"\b\d{1,2}:\d{2}\b", text)
+
+
+def _clean(v):
+    return v.replace(",", "").replace("%", "")
+
+
+def _vals(text):
+    return sorted({_clean(v) for v in (_nums(text) + _years(text) + _times(text))})
+
+
+def deterministic_verdict(doc):
+    """doc: the analyst prompt (claim + evidence). Returns a verdict dict."""
+    m = re.split(r"\bEvidence:\s*", doc, flags=re.IGNORECASE)
+    claim, evidence = m[0], (m[1] if len(m) > 1 else "")
+    cv = _vals(claim)
+    ev = _vals(evidence)
+    if not cv and not ev:
+        return {"verdict": "not enough information", "kind": "no-values",
+                "confidence": "LOW", "explain": "no numeric value to compare"}
+    if cv and not ev:
+        return {"verdict": "not enough information", "kind": "claim-only",
+                "confidence": "HIGH", "explain": "evidence has no numeric value to compare"}
+    if not cv:
+        return {"verdict": "unclear", "kind": "no-claim-value",
+                "confidence": "LOW", "explain": "claim has no numeric value"}
+    if set(cv) == set(ev):
+        return {"verdict": "supports", "kind": "equal", "confidence": "HIGH",
+                "explain": f"claim value {sorted(cv)} equals evidence value {sorted(ev)}"}
+    if not (set(cv) & set(ev)):
+        return {"verdict": "refutes", "kind": "differ", "confidence": "HIGH",
+                "explain": f"claim value {sorted(cv)} differs from evidence value {sorted(ev)}"}
+    return {"verdict": "unclear", "kind": "partial", "confidence": "LOW",
+            "explain": f"claim {sorted(cv)} partially overlaps evidence {sorted(ev)}"}

research/verify_loop.py

@@ -0,0 +1,172 @@
+"""External verification loop for the FSI suit (harness doctrine #5).
+
+Big-tech basis (docs/harness_research.md): LLMs cannot self-correct with
+intrinsic critique (arXiv 2310.01798); small models need STRONG EXTERNAL
+verifiers (arXiv 2404.09931); CRITIC makes tools the critic (2305.11738);
+Chain-of-Verification = draft -> verify -> revise (2309.09308).
+
+Flow (deterministic suit logic; the head never grades itself):
+  1. DRAFT: constrained analyst verdict on the claim (or rule spine first).
+  2. VERIFY: plan checkable value probes (numbers/years/times/quoted values)
+     from the claim; for each, retrieve the record evidence and run the
+     deterministic spine (research/verify.py).
+  3. REVISE: if the spine resolves (supports/refutes), the spine verdict
+     WINS (it cannot hallucinate); if unresolved (no-values/partial), keep the
+     draft verdict but do not raise confidence; if any check contradicts the
+     draft, downgrade to LOW and flag the discrepancy.
+  4. TRACE: full chain-of-custody (checks, sources, decisions).
+
+Usage:
+  from research.verify_loop import verify_case, plan_checks, run_checks
+"""
+import re
+
+from research.provenance import evaluate_source_policy
+from research.verify import deterministic_verdict
+
+_QUOTE = re.compile(r"[\"']([^\"']{4,60})[\"']")
+_NUM = re.compile(r"\b\d+(?:,\d{3})*(?:\.\d+)?%?\b")
+_YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
+_TIME = re.compile(r"\d{1,2}:\d{2}")  # no trailing \b: "9:30am" has none
+
+
+def plan_checks(claim):
+    """Extract checkable value probes from a claim (deterministic)."""
+    checks = []
+    seen = set()
+    for m in _QUOTE.finditer(claim):
+        q = m.group(1).strip()
+        if q.lower() not in seen:
+            seen.add(q.lower())
+            checks.append({"kind": "quote", "value": q})
+    # times first so their digits are not double-counted as numbers
+    for m in _TIME.finditer(claim):
+        v = m.group(0)
+        if v.lower() not in seen:
+            seen.add(v.lower())
+            checks.append({"kind": "time", "value": v})
+    rest = _TIME.sub(" ", claim)
+    for pat, kind in ((_NUM, "number"), (_YEAR, "year")):
+        for m in pat.finditer(rest):
+            v = m.group(0)
+            if v.lower() not in seen:
+                seen.add(v.lower())
+                checks.append({"kind": kind, "value": v})
+    return checks[:8]
+
+
+def _retrieval_bundle(raw):
+    """Normalize legacy text retrieval and traceable evidence bundles."""
+    if isinstance(raw, dict):
+        evidence = raw.get("evidence", raw.get("text", ""))
+        sources = raw.get("sources", [])
+        relation = str(raw.get("claim_relation", "")).strip().lower()
+        return str(evidence or ""), list(sources) if isinstance(sources, list) else [], relation
+    return str(raw or ""), [], ""
+
+
+def run_checks(checks, retrieve, spine=deterministic_verdict,
+               require_source_policy=False):
+    """Run each probe: retrieve evidence for the value, deterministic compare.
+
+    retrieve(value) -> text (legacy) or an evidence bundle:
+      {"evidence": str, "sources": [...], "claim_relation": supports|refutes}.
+    When ``require_source_policy`` is true, a bundle must pass SOP 09 and its
+    claim relation must agree with the deterministic value check. Otherwise it
+    is a lead, not verified evidence.
+    Returns list of {kind, value, evidence, verdict, kind_of_spine, explain}.
+    """
+    results = []
+    for c in checks:
+        evidence, sources, relation = _retrieval_bundle(retrieve(c["value"]))
+        if not evidence:
+            results.append({**c, "evidence": "", "verdict": "not enough information",
+                            "kind": "no-evidence", "explain": "no record retrieved"})
+            continue
+        doc = f"Claim: {c['value']}\nEvidence: {evidence}"
+        r = spine(doc)
+        result = {**c, "evidence": evidence[:200], "value_verdict": r["verdict"],
+                  "verdict": r["verdict"], "kind": r["kind"],
+                  "explain": r["explain"]}
+        if require_source_policy:
+            policy = evaluate_source_policy(sources)
+            result["source_policy"] = policy
+            result["source_ids"] = [card["source_id"] for card in policy["sources"]]
+            if not policy["verified"]:
+                result.update(verdict="not enough information", kind="source-policy-failed",
+                              explain=policy["reason"])
+            elif relation not in ("supports", "refutes"):
+                result.update(verdict="not enough information", kind="source-relation-missing",
+                              explain="verified source bundle lacks a checked claim relation")
+            elif relation != r["verdict"]:
+                result.update(verdict="not enough information", kind="source-relation-conflict",
+                              explain="claim relation conflicts with deterministic value check")
+            else:
+                result.update(verdict=relation, kind="source-policy-verified",
+                              explain="source policy and value check agree")
+        results.append(result)
+    return results
+
+
+def _classify(checks):
+    """Aggregate spine results: supports / refutes / mixed / unresolved."""
+    resolved = [c for c in checks if c["verdict"] in ("supports", "refutes")]
+    if not resolved:
+        return "unresolved", None
+    supports = sum(1 for c in resolved if c["verdict"] == "supports")
+    if supports == len(resolved):
+        return "supports", None
+    if supports == 0:
+        return "refutes", None
+    return "mixed", [c for c in resolved if c["verdict"] == "refutes"]
+
+
+def verify_case(claim, draft_verdict, draft_conf, retrieve,
+                spine=deterministic_verdict, require_source_policy=True):
+    """Draft -> verify -> revise. Returns a decision dict with trace.
+
+    claim: the claim under investigation (text).
+    draft_verdict/conf: the constrained analyst verdict + confidence.
+    retrieve(value) -> record evidence text or a traceable evidence bundle.
+    require_source_policy: fail closed unless the bundle passes SOP 09.
+    """
+    checks = plan_checks(claim)
+    results = run_checks(checks, retrieve, spine=spine,
+                         require_source_policy=require_source_policy)
+    status, refuting = _classify(results)
+
+    if status == "supports":
+        verdict, conf = "true", "HIGH"
+        basis = "source-policy-verified" if require_source_policy else "rule-verified"
+    elif status == "refutes":
+        verdict, conf, basis = "false", "HIGH", "rule-refuted"
+    elif status == "mixed":
+        verdict, conf = ("not enough information", "LOW") if require_source_policy else ("low confidence", "LOW")
+        basis = "rule-mixed:" + ",".join(c["value"] for c in refuting[:3])
+    else:
+        if require_source_policy:
+            verdict, conf, basis = "not enough information", "LOW", "source-policy-incomplete"
+            return {
+                "verdict": verdict,
+                "confidence": conf,
+                "basis": basis,
+                "checks": results,
+                "sources": [sid for c in results for sid in c.get("source_ids", [])][:8],
+                "abstained": True,
+            }
+        # unresolved: the spine cannot confirm; keep the draft but never raise
+        verdict = draft_verdict or "not enough information"
+        conf = draft_conf if draft_conf in ("LOW", "MEDIUM", "HIGH") else "LOW"
+        basis = "unresolved-by-spine"
+        if draft_conf == "HIGH":
+            conf, basis = "MEDIUM", "draft-high-downgraded-unverified"
+
+    return {
+        "verdict": verdict,
+        "confidence": conf,
+        "basis": basis,
+        "checks": results,
+        "sources": [sid for c in results for sid in c.get("source_ids", [])][:8]
+                   if require_source_policy else [c["evidence"] for c in results if c["evidence"]][:6],
+        "abstained": verdict == "not enough information",
+    }

research/websearch.py

@@ -0,0 +1,290 @@
+"""Live web / dark-web retrieval layer for the tiny researcher (client hands).
+
+The 16M model is the analyst brain and cannot browse. This module is the
+client-side retrieval tool: it searches the open web (Google News RSS,
+Internet Archive, Wikipedia - all key-free), optionally reaches .onion
+services through a local Tor SOCKS5 proxy, extracts documents into plain
+text, and writes them into the local library so TinyIndex can surface them
+for the model to verify, cross-check, and guide the user down rabbit holes.
+
+Guardrails (research/OSINT only):
+  * read-only, no identity, no credentials, no execution, size caps.
+  * http(s) and .onion only; other schemes (file:// ftp:// etc.) refused.
+  * .onion requests need a local Tor SOCKS proxy; if it is not running the
+    caller gets a clear, actionable message (never a silent empty result).
+"""
+
+from __future__ import annotations
+
+import html as _html
+import json
+import re
+import socket
+import ssl
+import time
+from pathlib import Path
+
+import requests
+import urllib.parse
+
+SOCKS_HOST = "127.0.0.1"
+SOCKS_PORT = 9050
+UA = "FSI-forensic-research/0.1 (research OSINT only) Mozilla/5.0"
+MAX_BODY = 1_500_000       # raw bytes cap per source
+MAX_CHARS = 120_000        # extracted text cap stored per doc
+TIMEOUT = (12, 25)         # requests (connect, read)
+
+
+def tor_status():
+    """Probe the local Tor SOCKS proxy. Returns (ok: bool, msg: str)."""
+    try:
+        with socket.create_connection((SOCKS_HOST, SOCKS_PORT), timeout=2):
+            return True, "Tor SOCKS is up on {0}:{1}".format(SOCKS_HOST, SOCKS_PORT)
+    except OSError:
+        return False, (
+            "Tor not reachable on {0}:{1}. Start a local Tor daemon "
+            "(tor, Tor Browser, or Orbot) and retry.".format(SOCKS_HOST, SOCKS_PORT)
+        )
+
+
+def _socks5_connect(host, port, timeout=5):
+    """Open a TCP socket to (host, port) through the local Tor SOCKS5 proxy."""
+    s = socket.create_connection((SOCKS_HOST, SOCKS_PORT), timeout=timeout)
+    try:
+        s.settimeout(timeout)
+        s.sendall(b"\x05\x01\x00")            # SOCKS5, 1 method: no-auth
+        rep = s.recv(2)
+        if rep != b"\x05\x00":
+            raise ConnectionError("Tor proxy requires auth (not supported)")
+        raw_host = host.encode("ascii", errors="ignore")
+        if len(raw_host) > 255:
+            raise ValueError("hostname too long for SOCKS5")
+        s.sendall(b"\x05\x01\x00\x03" + bytes([len(raw_host)]) + raw_host
+                  + port.to_bytes(2, "big"))
+        head = s.recv(512)                    # VER REP RSV ATYP ADDR BND.PORT
+        if len(head) < 2 or head[1] != 0x00:
+            raise ConnectionError("SOCKS5 CONNECT refused for {0}:{1}".format(host, port))
+        return s
+    except Exception:
+        s.close()
+        raise
+
+
+def _http_over_socks(host, port, https, path, timeout=25):
+    """Send one HTTP/1.1 GET over a Tor TCP socket (TLS-wrapped if https)."""
+    s = _socks5_connect(host, port, min(timeout, 10))
+    try:
+        if https:
+            ctx = ssl.create_default_context()
+            s = ctx.wrap_socket(s, server_hostname=host)
+        s.settimeout(timeout)
+        host_hdr = host if (https and port == 443) else "{0}:{1}".format(host, port)
+        req = ("GET {0} HTTP/1.1\r\nHost: {1}\r\nUser-Agent: {2}\r\n"
+               "Accept: text/html\r\nConnection: close\r\n\r\n").format(
+                   path, host_hdr, UA)
+        s.sendall(req.encode())
+        buf = b""
+        while len(buf) < MAX_BODY:
+            try:
+                chunk = s.recv(65536)
+            except (socket.timeout, OSError):
+                break
+            if not chunk:
+                break
+            buf += chunk
+        return buf
+    finally:
+        try:
+            s.close()
+        except Exception:
+            pass
+
+
+def _split_http(buf):
+    idx = buf.find(b"\r\n\r\n")
+    if idx < 0:
+        return buf, b""
+    return buf[:idx], buf[idx + 4:]
+
+
+def _extract_text(raw):
+    if isinstance(raw, (bytes, bytearray)):
+        txt = bytes(raw).decode("utf-8", errors="replace")
+    else:
+        txt = str(raw)
+    txt = re.sub(r"(?is)<(script|style|head|header|footer|nav)[^>]*>.*?</\1>", " ", txt)
+    txt = re.sub(r"(?i)<br\s*/?>[\r\n]*", "\\n", txt)
+    txt = re.sub(r"(?s)<[^>]+>", " ", txt)
+    txt = _html.unescape(txt)
+    txt = re.sub(r"[ \\t]+", " ", txt)
+    txt = re.sub(r"\n\s*\n+", "\n\n", txt)
+    return txt.strip()
+
+
+def _first_title(html_str):
+    m = re.search(r"(?is)<title[^>]*>(.*?)</title>", html_str)
+    if not m:
+        return "(untitled)"
+    return _extract_text(m.group(1))[:160] or "(untitled)"
+
+
+def fetch(url, tor=False, timeout=25):
+    """Fetch one URL (clearnet or .onion) into a dict with plain text."""
+    u = urllib.parse.urlparse(url)
+    if u.scheme not in ("http", "https"):
+        raise ValueError("refusing non-http(s) target: {0}".format(u.scheme))
+    is_onion = (u.hostname or "").endswith(".onion") or tor
+    if is_onion:
+        ok, msg = tor_status()
+        if not ok:
+            raise RuntimeError(msg)
+        port = u.port or (443 if u.scheme == "https" else 80)
+        path = (u.path or "/") + (("?" + u.query) if u.query else "")
+        raw = _http_over_socks(u.hostname, port, u.scheme == "https", path, timeout)
+        head, body = _split_http(raw)
+        content = _extract_text(body or raw)
+        status = re.search(br"HTTP/1\.[01] (\d{3})", head)
+        title = _first_title(raw.decode("utf-8", "replace"))
+        return {"url": url, "title": title, "content": content,
+                "source": "onion",
+                "status": (status.group(1).decode() if status else "?")}
+    r = requests.get(url, headers={"User-Agent": UA}, timeout=TIMEOUT,
+                     allow_redirects=True)
+    r.raise_for_status()
+    return {"url": r.url, "title": _first_title(r.text) or "(untitled)",
+            "content": _extract_text(r.content),
+            "source": "clearnet", "status": str(r.status_code)}
+
+
+def search_news(query, limit=10):
+    url = ("https://news.google.com/rss/search?q=" + urllib.parse.quote(query)
+           + "&hl=en-US&gl=US&ceid=US:en")
+    r = requests.get(url, headers={"User-Agent": UA}, timeout=TIMEOUT)
+    r.raise_for_status()
+    out = []
+    for item in re.findall(r"(?is)<item>(.*?)</item>", r.text)[:limit]:
+        t = re.search(r"(?is)<title>(.*?)</title>", item)
+        link = re.search(r"(?is)<link>(.*?)</link>", item)
+        desc = re.search(r"(?is)<description>(.*?)</description>", item)
+        pub = re.search(r"(?is)<pubDate>(.*?)</pubDate>", item)
+        if not link:
+            continue
+        out.append({
+            "title": (_html.unescape(t.group(1)) if t else "(news)").strip(),
+            "url": link.group(1).strip(),
+            "snippet": (_html.unescape(desc.group(1)).strip() if desc else ""),
+            "source": "google-news",
+            "date": (pub.group(1).strip() if pub else ""),
+        })
+    return out
+
+
+def search_archive(query, limit=5):
+    params = {"q": query, "fl[]": ["identifier", "title"],
+              "rows": limit, "output": "json"}
+    r = requests.get("https://archive.org/advancedsearch.php", params=params,
+                     headers={"User-Agent": UA}, timeout=TIMEOUT)
+    r.raise_for_status()
+    docs = r.json().get("response", {}).get("docs", [])
+    out = []
+    for d in docs:
+        if not d.get("identifier"):
+            continue
+        out.append({"title": (d.get("title") or d["identifier"]).strip(),
+                    "url": "https://archive.org/details/" + d["identifier"],
+                    "snippet": "Internet Archive item", "source": "archive-org",
+                    "date": ""})
+    return out[:limit]
+
+
+def search_wiki(query, limit=4):
+    p = {"action": "query", "list": "search", "srsearch": query,
+         "format": "json", "srlimit": limit}
+    r = requests.get("https://en.wikipedia.org/w/api.php", params=p,
+                     headers={"User-Agent": UA}, timeout=TIMEOUT)
+    r.raise_for_status()
+    res = r.json().get("query", {}).get("search", [])
+    out = []
+    for d in res:
+        out.append({
+            "title": d["title"],
+            "url": "https://en.wikipedia.org/wiki/" + urllib.parse.quote(
+                d["title"].replace(" ", "_")),
+            "snippet": re.sub(r"<.*?>", "", d.get("snippet", "")),
+            "source": "wikipedia", "date": ""})
+    return out
+
+
+def search_web(query, limit=10):
+    """Search clearnet across news + archive + wiki; dedup by URL."""
+    combined = []
+    for fn in (search_news, search_archive, search_wiki):
+        try:
+            combined += fn(query, limit=max(1, limit // 2 + 1))
+        except Exception:
+            continue
+    seen, out = set(), []
+    for r in combined:
+        if r["url"] in seen:
+            continue
+        seen.add(r["url"])
+        out.append(r)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def save_doc(library_dir, slug, title, text):
+    """Write one pulled document into the library as a .txt file. Returns Path."""
+    if not text:
+        raise ValueError("no text to save")
+    safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", slug)[:60].strip("_") or "doc"
+    p = Path(library_dir) / ("pull_{0}_{1}.txt".format(int(time.time()), safe))
+    p.write_text("TITLE: {0}\nSOURCE: {1}\n\n{2}".format(title, slug, text[:MAX_CHARS]),
+                 encoding="utf-8")
+    return p
+
+
+def pull(query, n=3, tor=False, library_dir="data/library"):
+    """Search, fetch the top-n docs, save each into the library. Returns dict."""
+    results = search_web(query, limit=max(n * 3, 6))
+    saved, errors = [], []
+    for r in results:
+        if len(saved) >= n:
+            break
+        try:
+            doc = fetch(r["url"], tor=tor)
+            if not doc["content"]:
+                errors.append({"url": r["url"], "err": "empty body"})
+                continue
+            p = save_doc(library_dir, r["url"].rsplit("/", 1)[-1], doc["title"],
+                         doc["content"])
+            saved.append({"url": r["url"], "title": doc["title"], "file": str(p)})
+        except Exception as e:
+            errors.append({"url": r["url"], "err": str(e)[:200]})
+    return {"query": query, "saved": saved, "errors": errors,
+            "tor": tor, "tor_status": tor_status()}
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser(description="live retrieval for the tiny researcher")
+    ap.add_argument("--search", help="search clearnet for a topic")
+    ap.add_argument("--fetch", help="fetch one URL")
+    ap.add_argument("--pull", help="search + fetch top docs into the library")
+    ap.add_argument("--tor", action="store_true", help="route fetches via Tor SOCKS")
+    ap.add_argument("--library", default="data/library")
+    ap.add_argument("--n", type=int, default=3)
+    args = ap.parse_args()
+
+    if args.search:
+        for r in search_web(args.search, limit=8):
+            print("[{0}] {1}\n    {2}\n    {3}".format(
+                r["source"], r["title"], r["url"], r["snippet"][:120]))
+    elif args.pull:
+        res = pull(args.pull, library_dir=args.library)
+        print(json.dumps(res, indent=2, ensure_ascii=False))
+    elif args.fetch:
+        print(json.dumps(fetch(args.fetch, tor=args.tor), indent=2, ensure_ascii=False))
+    else:
+        ap.print_help()

research/workspace.py

@@ -0,0 +1,167 @@
+"""Research workspace: turn case notes into analytic artifacts (the sandbox).
+
+The tiny head reasons; the workspace materializes. Given a case ledger (NOTE
+lines, search hits, verdicts) it renders, saves, and returns markdown documents:
+  - timeline      events with dates, sorted, source-tagged
+  - evidence      claim/evidence/value rows (the discrepancy table)
+  - series        two value series as an ASCII chart (pattern display)
+  - crossref      a theme/symbol mapped to every source that mentions it
+
+Deterministic only - no model inference here. Artifacts are saved under
+data/artifacts/ so a case produces durable documents, not just chat.
+"""
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+ART_DIR = ROOT / "data" / "artifacts"
+
+DATE_RE = re.compile(r"\b((?:19|20)\d{2}(?:-\d{1,2}(?:-\d{1,2})?)?)\b")
+VALUE_RE = re.compile(r"\b(\d{1,2}:\d{2}|\d+(?:,\d{3})*\.?\d*%?)\b")
+SOURCE_RE = re.compile(r"\[([a-z0-9_./-]+)\]|(https?://\S+)")
+
+ART_KINDS = ("timeline", "evidence", "series", "crossref")
+
+
+def _slug(s):
+    return re.sub(r"[^a-z0-9]+", "_", s.lower()).strip("_") or "artifact"
+
+
+def save_md(title, md):
+    ART_DIR.mkdir(parents=True, exist_ok=True)
+    path = ART_DIR / f"{_slug(title)}.md"
+    path.write_text(md + ("\n" if not md.endswith("\n") else ""), encoding="utf-8")
+    return path
+
+
+def _src(line):
+    m = SOURCE_RE.search(line)
+    return m.group(1) or m.group(2) if m else "-"
+
+
+def split_rows(ledger):
+    """Split a ledger into (date_rows, value_rows, bare_rows)."""
+    dates, values, bare = [], [], []
+    for ln in ledger:
+        ln = ln.strip()
+        if not ln:
+            continue
+        dm = DATE_RE.search(ln)
+        if dm:
+            dates.append((dm.group(1), ln, _src(ln)))
+            continue
+        vm = VALUE_RE.search(ln)
+        if vm:
+            values.append((ln, _src(ln)))
+        else:
+            bare.append(ln)
+    dates.sort(key=lambda r: r[0])
+    return dates, values, bare
+
+
+def render_timeline(ledger, title="Timeline"):
+    rows, _, _ = split_rows(ledger)
+    if not rows:
+        return None
+    out = [f"# {title}", "", "| Date | Event | Source |", "|---|---|---|"]
+    for d, ln, s in rows:
+        out.append(f"| {d} | {ln[:140]} | {s} |")
+    out.append("")
+    out.append("_Ordering is verifiable only where the source records the date; "
+               "gaps are as informative as entries._")
+    return "\n".join(out)
+
+
+def render_evidence(ledger, title="Evidence & Discrepancies"):
+    rows, _, _ = split_rows(ledger)
+    if not rows:
+        return None
+    out = [f"# {title}", "", "| Date | Statement | Source |", "|---|---|---|"]
+    for d, ln, s in rows:
+        out.append(f"| {d} | {ln[:140]} | {s} |")
+    return "\n".join(out)
+
+
+def render_series(series, title="Series Comparison"):
+    """series: list of (label, [numbers]). ASCII bars side by side."""
+    if not series or len(series) < 2:
+        return None
+    labels = [s[0] for s in series]
+    seqs = [list(s[1]) for s in series]
+    n = min(len(x) for x in seqs)
+    if n == 0:
+        return None
+    out = [f"# {title}", "", f"| {' | '.join(labels)} |", f"|{'---|' * len(labels)}"]
+    for i in range(n):
+        vals = [x[i] for x in seqs]
+        out.append("| " + " | ".join(f"{v:.4g}" for v in vals) + " |")
+    out.append("")
+    out.append("Points (index)  " + "   ".join(f"[{i}]" for i in range(n)))
+    for j, (lab, seq) in enumerate(series):
+        mx = max(seq) or 1
+        bars = ["#" * max(1, round(v / mx * 20)) for v in seq]
+        out.append(f"{lab}: " + " ".join(bars))
+    out.append("")
+    out.append("_The chart only compares values; it asserts nothing about cause._")
+    return "\n".join(out)
+
+
+def render_crossref(theme, lines, title="Cross-Reference"):
+    """theme: a term/symbol; lines: source-tagged ledger/notes."""
+    out = [f"# {title}", "", f"Theme/symbol: **{theme}**", "", "| Source | Context |", "|---|---|"]
+    hit = 0
+    low = theme.lower()
+    for ln in lines:
+        if low in ln.lower():
+            out.append(f"| {_src(ln)} | {ln[:150]} |")
+            hit += 1
+    if not hit:
+        out.append("| - | (no mention in this case's documents) |")
+    out.append("")
+    out.append("_Absence of a mention is a finding, not an error: note it explicitly._")
+    return "\n".join(out)
+
+
+def synthesize(ledger, title, series=None, theme=None, lines=None):
+    """Compose every artifact available for a case into one saved document."""
+    arts = []
+    tl = render_timeline(ledger, title=f"{title} - Timeline")
+    if tl:
+        arts.append(tl)
+    ev = render_evidence(ledger, title=f"{title} - Evidence & Discrepancies")
+    if ev:
+        arts.append(ev)
+    ch = render_series(series, title=f"{title} - Series Comparison") if series else None
+    if ch:
+        arts.append(ch)
+    cr = render_crossref(theme, lines or ledger, title=f"{title} - Cross-Reference") if theme else None
+    if cr:
+        arts.append(cr)
+    if not arts:
+        return None
+    doc = "\n\n---\n\n".join(arts)
+    ART_DIR.mkdir(parents=True, exist_ok=True)
+    path = ART_DIR / f"{_slug(title)}.md"
+    path.write_text(doc + "\n", encoding="utf-8")
+    return str(path), doc
+
+
+def parse_series_arg(arg):
+    """'title | label1:1,2,3 | label2:4,5,6' -> (title, [(label, [nums])])."""
+    parts = [p.strip() for p in arg.split("|")]
+    title = parts[0] or "Series Comparison"
+    series = []
+    for p in parts[1:]:
+        if ":" not in p:
+            continue
+        lab, vs = p.split(":", 1)
+        nums = []
+        for v in vs.replace(" ", "").split(","):
+            try:
+                nums.append(float(v))
+            except ValueError:
+                pass
+        if nums:
+            series.append((lab.strip(), nums))
+    return title, series

run_code.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_lm.py \
+  --data data/code_train.bin --val data/code_valid.bin --tok data/tokenizer.json \
+  --config tiny10m --ckpt ckpt/code \
+  --resume ckpt/forensic --batch 16 --seq 256 --lr 2e-4 --warmup 100 --steps 2500 \
+  --eval-every 250 --save-every 500 --threads 8 --seed 1 \
+  > logs/code_train.log 2>&1 < /dev/null &
+echo "code launched pid $!"

run_distill.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_sft.py \
+  --base ckpt/forensic --data data/sft_distill_mix.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/distill --epochs 30 --batch 8 --seq 256 --lr 3e-5 \
+  --eval-every 40 --threads 8 \
+  > logs/distill_train.log 2>&1 < /dev/null &
+echo "distill launched pid $!"

run_domain_adapt.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD" OMP_NUM_THREADS=4 MALLOC_ARENA_MAX=2
+.venv/bin/python -u train/train_lm.py \
+  --init-from ckpt/nlp_full --data data/domain_full.bin --val data/domain_valid.bin \
+  --config tiny10m --ckpt ckpt/nlp_domain \
+  --batch 16 --seq 256 --lr 1e-4 --min-lr 1e-5 --warmup 50 \
+  --steps "${STEPS:-1600}" --log-every 25 --eval-every 200 --save-every 200 \
+  --threads 4 2>&1 | tee logs/train_domain.log

run_dpo.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_dpo.py \
+  --base ckpt/distill --data data/prefs_persona.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/dpo --epochs 6 --batch 4 --seq 256 --lr 1e-5 --beta 0.1 \
+  --threads 8 \
+  > logs/dpo_train.log 2>&1 < /dev/null &
+echo "dpo launched pid $!"

run_dpo_sop.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+# combined preference set: persona style + procedure-following
+.venv/bin/python - << 'PY'
+import json
+seen, out = set(), []
+for src in ("data/prefs_persona.jsonl", "data/prefs_sop.jsonl"):
+    for line in open(src, encoding="utf-8"):
+        line = line.strip()
+        if not line:
+            continue
+        ex = json.loads(line)
+        k = ex["prompt"]
+        if k in seen:
+            continue
+        seen.add(k)
+        out.append(ex)
+with open("data/prefs_all.jsonl", "w", encoding="utf-8") as f:
+    for ex in out:
+        f.write(json.dumps(ex) + "\n")
+print(f"prefs_all.jsonl: {len(out)} pairs")
+PY
+setsid ./.venv/bin/python train/train_dpo.py \
+  --base ckpt/sop --data data/prefs_all.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/dpo --epochs 6 --batch 4 --seq 256 --lr 1e-5 --beta 0.1 \
+  --threads 8 \
+  > logs/dpo_sop.log 2>&1 < /dev/null &
+echo "dpo launched pid $!"

run_nlp.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+nohup ./.venv/bin/python train/train_lm.py \
+  --data data/train.bin --val data/valid.bin --tok data/tokenizer.json \
+  --config tiny10m --ckpt ckpt/nlp \
+  --batch 16 --seq 256 --lr 3e-4 --warmup 300 --steps 7000 \
+  --eval-every 500 --save-every 1000 --threads 8 --seed 42 \
+  > logs/nlp_train.log 2>&1 &
+echo "launched pid $!"

run_nlp2.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_lm.py \
+  --data data/train.bin --val data/valid.bin --tok data/tokenizer.json \
+  --config tiny10m --ckpt ckpt/nlp --resume ckpt/nlp \
+  --batch 32 --seq 128 --lr 2e-4 --warmup 100 --steps 1200 \
+  --log-every 20 --eval-every 300 --save-every 300 --threads 8 --seed 3 \
+  > logs/nlp2_train.log 2>&1 < /dev/null &
+echo "nlp2 launched pid $!"

run_nlp3.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_lm.py \
+  --data data/train2.bin --val data/valid.bin --tok data/tokenizer.json \
+  --config tiny10m --ckpt ckpt/nlp --resume ckpt/nlp \
+  --batch 32 --seq 128 --lr 1.5e-4 --warmup 100 --steps 1500 \
+  --log-every 20 --eval-every 300 --save-every 300 --threads 8 --seed 5 \
+  > logs/nlp3_train.log 2>&1 < /dev/null &
+echo "nlp3 launched pid $!"

run_pipeline.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Full on-device chain, each stage waits for the previous:
+#   nlp2 pretrain (fixed arch, already running) -> forensic SFT -> SOP SFT -> DPO
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+export PATH="$PWD/.venv/bin:$PATH"
+
+echo "waiting for running pretrain (train/train_lm.py) to finish..."
+while pgrep -f "train/train_lm.py" > /dev/null; do sleep 30; done
+echo "pretrain done."
+
+echo "[1/3] forensic SFT (fixed-architecture base)..."
+python train/train_sft.py \
+  --base ckpt/nlp --data data/sft_forensic.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/forensic --epochs 1 --batch 8 --seq 256 --lr 5e-5 \
+  --eval-every 200 --threads 8 > logs/sft_train.log 2>&1
+echo "[2/3] SOP SFT (forensic + teacher distill + procedures + room actions)..."
+python train/train_sft.py \
+  --base ckpt/forensic --data data/sft_sop_mix.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/sop --epochs 12 --batch 8 --seq 256 --lr 3e-5 \
+  --eval-every 40 --threads 8 > logs/sop_train.log 2>&1
+echo "[3/3] DPO (persona style + procedure-following preference)..."
+python - << 'PY'
+import json
+seen, out = set(), []
+for src in ("data/prefs_persona.jsonl", "data/prefs_sop.jsonl"):
+    for line in open(src, encoding="utf-8"):
+        line = line.strip()
+        if not line:
+            continue
+        ex = json.loads(line)
+        if ex["prompt"] in seen:
+            continue
+        seen.add(ex["prompt"])
+        out.append(ex)
+with open("data/prefs_all.jsonl", "w", encoding="utf-8") as f:
+    for ex in out:
+        f.write(json.dumps(ex) + "\n")
+print(f"prefs_all.jsonl: {len(out)} pairs")
+PY
+python train/train_dpo.py \
+  --base ckpt/sop --data data/prefs_all.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/dpo --epochs 6 --batch 4 --seq 256 --lr 1e-5 --beta 0.1 \
+  --threads 8 > logs/dpo_sop.log 2>&1
+echo "pipeline complete -> ckpt/dpo"
+echo "next: research/probe.py --ckpt ckpt/dpo ; then code stage (./run_code.sh after pointing its base at ckpt/dpo)"

run_pretrain_full.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD" OMP_NUM_THREADS=4 MALLOC_ARENA_MAX=2
+.venv/bin/python -u train/train_lm.py \
+  --resume ckpt/nlp_full --data data/train_full.bin --val data/valid.bin \
+  --config tiny10m --ckpt ckpt/nlp_full \
+  --batch 16 --seq 256 --lr 1.5e-4 --min-lr 1e-5 --warmup 200 \
+  --steps "${STEPS:-5000}" --log-every 25 --eval-every 500 --save-every 500 \
+  --threads 4 2>&1 | tee -a logs/train_full.log

run_sft.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_sft.py \
+  --base ckpt/nlp --data data/sft_forensic.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/forensic --epochs 3 --batch 8 --seq 256 --lr 5e-5 \
+  --eval-every 200 --threads 8 \
+  > logs/sft_train.log 2>&1 < /dev/null &
+echo "sft launched pid $!"

run_sop.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+setsid ./.venv/bin/python train/train_sft.py \
+  --base ckpt/forensic --data data/sft_sop_mix.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/sop --epochs 12 --batch 8 --seq 256 --lr 3e-5 \
+  --eval-every 40 --threads 8 \
+  > logs/sop_train.log 2>&1 < /dev/null &
+echo "sop sft launched pid $!"

run_tui.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD"
+CKPT="${1:-ckpt/v15_lora/best.pt}"
+exec ./.venv/bin/python tui/analyst.py --ckpt "$CKPT"

run_v2.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD" OMP_NUM_THREADS=8
+exec ./.venv/bin/python -u train/train_sft2.py \
+  --base ckpt/nlp --data data/sft_mix_v2.jsonl --tok data/tokenizer.json \
+  --ckpt ckpt/v2 --epochs 3 --batch 8 --seq 256 --lr 2e-5 \
+  --eval-every 25 --log-every 25 --threads 8

skills/tiny-model-agent-notes/SKILL.md

@@ -0,0 +1,42 @@
+---
+name: tiny-model-agent-notes
+description: The living-project-record discipline for FSI Anomaly — every change, trial, measurement, and decision is recorded in agent_notes.md (and CHANGELOG.md) with dates and honest numbers, so sessions/agents stay in sync. The notes also feed the end-of-project training documents, the war story, and the paper. Use whenever anything changes on the project (run started/finished, gate result, code change, data authored, decision made) and when auditing project history.
+---
+
+# Tiny-Model Agent Notes — the living record
+
+## Purpose
+`agent_notes.md` (repo root) is the single chronological record of the project:
+what we tried, what worked, what didn't, the research behind every decision,
+and where we stand. It keeps every session and every collaborating agent in
+sync, and it becomes part of the final training-document set, the war story,
+and the paper.
+
+## Update discipline (MANDATORY, after EVERY action)
+1. Any change to the project — run started/finished, gate result, code change,
+   data authored, hyperparameter tried, decision made — MUST be appended to
+   `agent_notes.md` AND `CHANGELOG.md` with the date.
+2. Entries are factual and measurable: numbers, paths, commands, verdicts.
+   No silent re-rolls — record the failure first, then the next attempt.
+3. `agent_notes.md` sections stay current:
+   - "Where we stand right now" is updated at the end of every working session.
+   - The timeline grows chronologically (append, don't rewrite history).
+   - Scorecard tables get the new row with the honest numbers.
+   - Open questions / not-yet-tried is pruned when a question is answered.
+4. Skills get updated when a finding becomes a RULE (e.g., replay mandatory) —
+   the notes record what happened; the skill encodes what to do next time.
+5. Cross-check: if a session starts and `agent_notes.md` is stale (does not
+   match the last CHANGELOG entry), reconcile it FIRST.
+
+## What the notes feed
+- Session continuity for human + AI collaborators (the user's models keep up
+  between sessions by reading this file).
+- The end-of-project TRAINING documents (owner directive: the notes are part
+  of the training data at the end).
+- The WAR STORY and PAPER: the honest engineering narrative (what was tried,
+  what the research said, what the measurements showed, what we learned).
+
+## Changelog
+- 2026-08-09: created with the full project history compiled from
+  CHANGELOG.md + skills; agent_notes.md written at repo root (249 lines);
+  device RAM+ change measured and recorded (swap 4.0G -> 12.3G).

skills/tiny-model-arch/SKILL.md

@@ -0,0 +1,122 @@
+---
+name: tiny-model-arch
+description: Architecture research + dry-run guardrails for the FSI tiny-liquid models — decides WHETHER and HOW to change the 7.8-13M liquid architecture (MoE / nanobot-experts / width / depth / attention) before committing training time. Use whenever choosing the model architecture, testing a "punch above weight" idea, or evaluating MoE/compression on edge hardware. Encodes measured results from this device so the same experiments are never re-run.
+---
+
+# Tiny-Model Architecture (7.8-13M, measured on THIS tablet)
+
+Goal: raise tiny-model capability without growing the footprint. BEFORE changing
+architecture, run the cheap dry-runs below — never gamble a multi-day training run
+on an un-proven arch.
+
+## Measured device / baseline (Aug 2026)
+8-core ARMv9, SVE2/BF16, ~2.4GB free RAM, ~780+ tok/s at d=320 with BF16. Baseline
+`ckpt/nlp_full` val loss ~2.58; v15 LoRA (7.8M) is the current best analyst.
+
+## Dry-run battery (run each before any arch change)
+
+### 1. Throughput + real params (60-step training smoke, fp32, batch8 seq128, lr 3e-4)
+Measured on data/train2.bin (loss @60 steps):
+| config | realM | tok/s | loss@60 | experts used |
+|---|---|---|---|---|
+| dense6 (dense, 6 blocks) | 7.79 | 1003 | 5.326 | n/a |
+| nano32 (32x~10K experts, top2) | 4.90 | 809 | 5.522 | 27/32 |
+| nano64 (64x~10K experts, top2) | 6.93 | 577 | 5.474 | 51/64 |
+| nano250 (250x~10K experts, top2) | 7.99 | 744 | 5.398 | 51/250 |
+FINDINGS:
+- Dense still reaches the lowest loss per-step at this scale. MoE does NOT outlearn
+  dense in a 60-step smoke; its benefit is capacity-per-FLOP at training time, not
+  faster convergence.
+- ROUTER COLLAPSE (load imbalance): nano250 uses only 51/250 experts after 60 steps.
+  The router cannot learn to spread tokens over 250 tiny experts with plain SFT.
+  Fixes that work in the field: an auxiliary load-balancing loss (Switch/Mixtral/
+  DeepSeek), or shared experts + few routed experts (DeepSeek-V2 style).
+- nano64 throughput DROPS (577) vs nano32/nano250: naive per-token python batching
+  costs more than tiny-expert compute. Vectorize routing/combine before expecting wins.
+- CPU-autocast bug: `router(x)` on a 3D bf16 input fails under torch CPU autocast.
+  Cast the router input to float (or run fp32) — the smallest MoE fix.
+
+### 2. Density / compressibility math
+nanobot-MoE reaches ~0.04-0.14% compute density (activated params per token) — the
+"how dense/compress" answer is: extremely sparse is possible, but only learnable with
+(a) a load-balance loss and (b) enough data per expert. 250 x ~10K-param experts needs
+~2.5M params/layer just for experts — NOT compatible with ~13M total in several layers.
+Compression levers: (1) shared-base matrices + per-expert LoRA deltas (many cheap
+experts), (2) low-rank factorized experts (W=A.B), (3) bf16 now / int8+ at inference.
+Router is trivial to compress (d x E).
+
+## Recommendations (verified direction, not promises)
+- AVOID 250-tiny-expert top-2 as the default: collapse + no per-step win on this device.
+- PREFER: dense depth-growth to 13M (proven, baseline-preserving, skill tiny-model-phase2)
+  OR a small MoE (8-16 experts) WITH a load-balancing loss if sparsity is the goal.
+- For a genuine 20M+ later: wide-head tower or shared-expert MoE require prototyping and
+  a head-to-head probe test; do NOT change architecture until the new arch beats dense in
+  an 8k-step curriculum + probe eval (verdict accuracy + value citation).
+
+## Gate before any real arch-change run
+The proposed arch must (a) match dense per-step loss in a 1k-step smoke, (b) use most of
+its experts (>=75%) with a load-balance loss, (c) train faster or equal on this tablet,
+and (d) beat dense on the probe battery after an 8k-step curriculum SFT. Any arch that
+fails (b) is rejected regardless of param count.
+
+## NEW: persona-routed sparse experts (dual-mind in ONE model) — measured, promising
+Idea: fuse dual-mind into a single forward pass by routing on the persona token
+(<|analyst|> vs <|skeptic|>) so each "mind" activates its own expert group. No 2x
+inference passes; one liquid trunk + sparse MoE whose router is persona-aware.
+Dry-run (16 experts, top-2, 160 steps on persona-tagged analyst+skeptic data):
+- usage correlation between analyst and skeptic inputs = 0.435 (1.0 = identical
+  routing; <1 = divergence) => the router DID learn partial persona specialization
+  without any architectural change (persona token flows through the liquid state
+  into the router).
+- e.g. expert 8/11/14 leaned analyst, expert 0/1/3/4/13 leaned skeptic.
+- Sharpen next with an explicit persona->router bias term (target corr < 0.3).
+This is the strongest "combine the concepts" candidate: liquid recurrence + sparse
+experts + dual minds, one model, one forward pass.
+
+
+## Liquid scan numerics (measured)
+Chunked log-space scan must keep exp args < 709: SCAN_CHUNK=16 (max arg 442) is
+the verified safe setting; 128 overflows to NaN when gates saturate. Any future
+recurrence rewrite must preserve the bounded-exponent property.
+
+## Vocabulary & Context decision record (2026-08-08) — researched, applied
+Question: bigger vocab (currently 8192) and longer/"unlimited" context for the 25.4M head.
+
+### Research basis (primary sources)
+- LFM2 Technical Report (Liquid AI, arXiv 2511.23404): ALL sizes 350M-8.3B ship with
+  32K context; compact hybrid = gated short convs + a FEW grouped-query-attention blocks;
+  training = curriculum (difficulty-ordered) -> SFT -> length-normalized preference
+  optimization -> model merging; CPU-first, on-device focus. => context 32k is standard at
+  ANY size, but Liquid's own small models keep a small number of attention blocks for it.
+- Position Interpolation (Chen et al 2023, arXiv 2306.15595): RoPE models extend context
+  with ~1k fine-tune steps; quality preserved in-window. Applies to us only if we keep RoPE.
+- Mamba (Gu & Dao 2023, arXiv 2312.00752) + Gated Linear Attention (Yang et al 2024,
+  arXiv 2312.06635): linear-recurrent/SSM layers have O(1) memory per token; long context
+  costs FLOPs, not KV cache. Our liquid scan (SCAN_CHUNK=16, bounded exp) already has this
+  property; 1024-token inference fits in the recurrent state.
+- TinyStories (Eldan & Li 2023, arXiv 2305.07759): 28M models are coherent ONLY when the
+  data is constrained/simple; fluency is a data property, not a scale property.
+- Vocab reference points: GPT-2 50,257 (byte BPE); Llama 32,000; SmolLM2 49,152;
+  Qwen2.5 151,936 (multilingual); Phi-1 51,200. No serious tiny model ships 8k.
+
+### Measured on THIS model
+- vocab 8192 x d_model 320, tied embeddings = 2.62M params = 10.3% of 25.43M.
+- 16k vocab -> 5.24M (20.6%); 32k -> 10.49M (41.2%): 32k is rejected at this size.
+- max_seq_len 1024 in config; SFT trains at seq 512; val at seq 64.
+- Corpus char coverage: the 806-row forensic corpus uses 79 distinct chars, all covered.
+  8k vocab is NOT the bottleneck for training data; it only hurts unseen named entities
+  (onion URLs, usernames, dates) in live use — and no small BPE handles those gracefully.
+
+### Decisions
+1. VOCAB: KEEP 8192 for the current baseline. 16k is the ceiling and only as a full
+   retokenize + embedding-expand + continue-pretrain job (multi-hour, later phase).
+   Do not retrain the tokenizer for a mid-life swap.
+2. CONTEXT: keep max_seq_len 1024. "Unlimited context" is a SUIT problem (retrieval +
+   memory slots + entity tracking — research/orchestrator.py, library, helix memory),
+   not a weights problem for a 25M head. The honest effective reasoning horizon is a few
+   hundred tokens; retrieval is the multiplier. Long-doc continue-pretrain at seq 1024 is
+   the weights-side upgrade path (next long run after fluency is fixed).
+3. FLUENCY IS THE BIGGER LEVER: measured free-form generation is template-loop garbage at
+   pretrain, SFT, AND DPO checkpoints while constrained-decode probes work. Per TinyStories
+   + training skill Phase-2, the fix = mix handcrafted fluent dialogue back into SFT
+   (<=15-20% of rows) from the best probe checkpoint. See tiny-model-training apply plan.

skills/tiny-model-deploy/SKILL.md

@@ -0,0 +1,45 @@
+---
+name: tiny-model-deploy
+description: Deployment/client guardrails for the FSI tiny-liquid researcher models — the analyst TUI (Parrot-OS-meets-Matrix theme), inference-time features (self-consistency majority vote, BM25 retrieval, source-DNA/provenance tool), and safe dark-web/OSINT operations (Tor, .onion verification, PGP, no-identity rules). Use whenever building or changing the client, tool layer, or inference loop around the tiny researcher model.
+---
+
+# Tiny-Model Deploy — client + tool layer guardrails
+
+The model is the ANALYST BRAIN; the client is the HANDS (retrieval, navigation,
+verification tooling). Keep the split clean: nothing in the client changes the
+model's weights; everything the client does must be honest about provenance.
+
+## Interface
+- `tui/analyst.py` — curses TUI, opencode-style agent, Parrot-OS-meets-Matrix
+  neon (matrix green, pink, blue, purple on near-black). Chat personas, SOP
+  cases, library search, case files.
+- `run_tui.sh` — launcher; default ckpt must point at the CURRENT best analyst.
+- `generate.py` — one-shot generation for tests/probes.
+
+## Inference-time features (no training change)
+1. SELF-CONSISTENCY (proven, 2203.11171): sample N verdicts (N>=3), majority
+   vote on the verdict class; cite the majority reasoning. Cheap reliability
+   boost at inference. Implement in the client, not the model.
+2. BM25 RETRIEVAL: local index over the user's document folder; client retrieves
+   top-k, model analyzes only retrieved text. No web memory in the model.
+3. PROVENANCE TOOL: compute/attach source-DNA tags (primary/secondary/anonymous,
+   independent-origin count, hash/PGP status) before the model sees a document.
+   Model ranks verifiable-vs-blotchy from the tags; client verifies the hashes.
+4. SAFE DARK-WEB OPS (client side): Tor Browser sessions, .onion address
+   verified against a trusted published mirror, PGP signature checks, no
+   downloads/JS/logins, disposable identity only. The client refuses unsafe
+   actions (opening unverified files, executing downloads).
+
+## Guardrails
+- Never auto-open unverified files or execute downloads from .onion channels.
+- Never let the client send real credentials/identity to any .onion service.
+- Always show the user the source and its verification status with every claim.
+- Keep the TUI dependency-free (stdlib curses; zero third-party deps).
+- If the model's output is not parseable (missing Verdict line), the client
+  shows the raw text and flags "unparsed" rather than faking a verdict.
+## Changelog
+- 2026-08-05: created; interface, inference features (self-consistency, BM25,
+  provenance tool, safe dark-web ops), guardrails.
+- 2026-08-05: added live retrieval layer (research/websearch.py) + engine/CLI
+  (/web /fetch /pull /tor) so the system can search the web and dark web (.onion
+  via a local Tor SOCKS proxy) and pull documents into the library.

skills/tiny-model-developer-credo/SKILL.md

@@ -0,0 +1,108 @@
+---
+name: tiny-model-developer-credo
+description: THE always-on discipline doctrine for the FSI tiny-liquid project — the Mandalorian creed ("This is the Way") translated into development guardrails: verify everything, never guess, surgical accuracy, absolute-highest-quality artifacts, no half-measures, no synthetic training data. Applies to every file, every training document, every stage. Use at the start of every task and during every gate.
+---
+
+# The Developer's Credo — Always-On Discipline
+
+> "We verify everything we do. We research everything. We do not guess. We do
+> not go in circles. Surgical accuracy and precision is how we stay in line.
+> We only use what we have verified — and the highest-quality, most efficient
+> path — on everything we do." — the Owner (2026-08-12)
+
+The Mandalorians are disciplined not because of their armor but because of
+their Way. This is our Way, translated from their creed to engineering.
+
+## The Six Actions (Resol'nare -> Engineering)
+
+1. **Wear armor.** The skills and SOPs ARE the armor. Every task opens with the
+   discipline loop: research -> skill -> apply -> gate -> measure -> record.
+   The armor is never off.
+2. **Speak the language.** Record everything precisely: agent_notes.md,
+   CHANGELOG.md, measured numbers. Mando'a = naming things exactly; no vague
+   "feels better" claims. Numbers only.
+3. **Defend the family.** Protect the user, their data, and the mission: PII
+   guardrails, source protection, honest abstention. A Mandalorian never harms
+   an innocent; the model never fabricates or leaks.
+4. **Raise the children as Mandalorians.** Every artifact (document, gold row,
+   preference pair, checkpoint, skill) is created to the same standard so the
+   next stage inherits discipline. Leave nothing half-raised.
+5. **Contribute to the clan.** Give back: honest eval cards, open pipeline
+   (Apache-2.0), the community gets the measured truth, not marketing.
+6. **Rally when called.** Gates fire without hesitation: ppl abort, calibration
+   thresholds, red-team failures stop the line. "Mandalorians don't run" — not
+   from a failing run, not from a hard gate, not from an inconvenient number.
+
+## The Core Sayings (translated)
+
+- **"This is the Way."** -> The SOP loop IS the Way. Every step of every stage:
+  research -> skill -> apply -> gate -> measure -> record.
+- **"Mandalorians don't run."** -> No shortcuts, no abandoning a broken run
+  without diagnosis, no blaming the environment. Diagnose, fix, resume.
+- **"I have spoken."** -> Every claim is backed by evidence or it is not spoken.
+  Abstain when the record is silent: "cannot confirm" beats speculation.
+- **"I am a Mandalorian. Weapons are part of my religion."** -> Tools (harness,
+  code, skills) are part of the discipline, not separate from it. The suit
+  amplifies the skill; it never replaces it.
+
+## The Absolute Quality Bar (training documents)
+
+Every training document, gold row, preference pair, prompt, and evaluation
+example is produced at the absolute highest quality we can produce. Period.
+
+- **No half-ass.** If a row is not production-grade, it does not enter the set.
+  Quality > quantity, always (Phi-1 "Textbooks Are All You Need", LIMA;
+  see tiny-model-kd).
+- **No synthetic rows, no generators, no scripts** for training data
+  (tiny-model-kd hard rule). Handcrafted, verifiable, teacher-authored only.
+- Every row must teach the model something true and checkable.
+- Treat every model as if it were your own software and your life depended on
+  it. This is a career and a grant application, not a toy.
+
+## The Discipline SOP (always on)
+
+1. Verify everything we do.
+2. Research everything — from multiple sources; then create/update the skill;
+   then apply it.
+3. No guessing; no going in circles; never re-run a measured dead end
+   (tiny-model-sop, tiny-model-roadmap).
+4. Surgical accuracy and precision in every edit.
+5. Measure and record everything (tiny-model-agent-notes, tiny-model-tracking).
+
+## The Credo Applied to Architecture (growth decisions)
+
+### Core Principles
+
+1. **Grow Width Before Depth** — "Efficient width scaling beats depth at small
+   scale" (EfficientNet arXiv:1905.11970, MLP-Mixer 2021). Tower widens
+   correctly; trunk depth FAILED (tiny-model-phase2 measured).
+2. **Preserve the Baseline** — "Never destroy what works" (LoRA arXiv:2106.09685,
+   Phi-1.5). Identity-init tower expansion; fluent base never modified.
+3. **Compartmentalize New Capacity** — "Isolation prevents drift" (MoE
+   arXiv:2006.16692, AdapterFusion 2021). Tower is isolated; trunk stays
+   fluency-preserving.
+4. **The Harness Scales Better Than the Model** — "Tools > params" (Toolformer
+   2023, tiny-model-suit). Build the suit, then grow the brain.
+5. **Measure, Don't Guess** — "Every growth step verified" (Chinchilla
+   arXiv:2205.05131, tiny-model-eval). Baseline preserved; PPL guard 60.
+6. **Width via Tower Expansion** — trunk (320, frozen) -> up_proj (identity)
+   -> tower (800, identity-init) -> down_proj (zero-init).
+
+### Implementation Rules
+1. grow_weights.py --verify must show baseline preserved.
+2. Frozen base + LoRA adapters only.
+3. Identity-init new tower weights.
+4. PPL guard: abort if > 60.0.
+5. Replay ratio 0.5 (arXiv 2502.06042).
+6. Checkpoint every 50 steps.
+
+## Build/audit rules
+1. Consult this skill at the START of every task and during every gate.
+2. Any work that violates the Quality Bar or skips the SOP loop is rejected.
+3. Changelog every application.
+
+## Changelog
+- 2026-08-12: Consolidated tiny-model-mandalorian into this skill (owner
+  renamed it to the Developer's Credo). Added Absolute Quality Bar, always-on
+  framing, and the Mandalorian creed translation.
+- 2026-08-09: Created with architecture growth principles.