scripts/build_notebook.py

#!/usr/bin/env python3
"""Build the Stoicheia demo notebook (anonymous, Colab-ready)."""
import json, pathlib

ORG = "anonymous-stoicheia"

def md(*lines):  return {"cell_type": "markdown", "metadata": {}, "source": [l + "\n" for l in lines]}
def code(*lines): return {"cell_type": "code", "metadata": {}, "execution_count": None,
                          "outputs": [], "source": [l + "\n" for l in lines]}

cells = [
 md("# Stoicheia — a character-level model for Ancient Greek",
    "",
    "Stoicheia is a 405M-parameter character-level masked-diffusion encoder for Ancient Greek.",
    "Its input is factored into five aligned planes — letters, word/sentence boundaries,",
    "diacritics, capitalization, punctuation — and **any of them can be set to *unknown* at",
    "inference**. One model therefore reads an edited text, bare *scriptio continua*, and a",
    "lacuna of unknown length, changing nothing but its input.",
    "",
    "This notebook runs the whole release end to end on a free Colab GPU (CPU works too, slower):",
    "",
    "1. restore a lacuna whose width is *not known* in advance",
    "2. pick the restoration model that has provably **never read** your document",
    "3. tag and parse a verse of Homer",
    "4. macronize and scan a line of verse",
    "5. score the macronizer against a hand-annotated benchmark",
    "",
    "Every model and dataset used below is public. Anonymous release accompanying a paper under review."),

 code("%pip install -q --upgrade transformers huggingface_hub safetensors torch datasets"),

 md("## 1. Restoring a lacuna of unknown width",
    "",
    "The hard case in epigraphy and papyrology is a break whose extent is uncertain, in text that",
    "carries no accents and no word division. Write `[N±M]` and the model scores every width in",
    "`N-M … N+M` by its own confidence, restoring the letters, the accents and the word boundaries",
    "together."),
 code("import sys, torch",
      "from transformers import AutoModel",
      "from huggingface_hub import snapshot_download",
      "",
      f'REPO = "{ORG}/Stoicheia-doc_clean"   # zero exposure to inscriptions or papyri',
      'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])',
      "sys.path.insert(0, local)",
      "",
      "model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()",
      "from processing_char_bert import CharBertProcessor",
      "proc = CharBertProcessor()",
      "",
      "# John 1:1 as it would reach us on a damaged, unaccented, unspaced witness",
      'damaged = "εναρχηηνο[5±3]καιολογοςηνπροστονθεον"',
      "best, width, candidates = proc.restore_elastic(model, damaged, mask_dia_boundary=True)",
      'print("restored :", best)',
      'print("width    :", width, "characters")',
      'for c in candidates[:5]:',
      '    print("   ", c)'),

 md("## 2. The model that has never read your document",
    "",
    "A single fixed train/test split makes a model useless for exactly the documents an editor",
    "cares about. Ten restoration checkpoints are released instead, one per held-out final digit",
    "of the PHI/TM identifier: whatever inscription or papyrus you are working on, one of the ten",
    "has provably never seen it during fine-tuning, and its backbone never saw a documentary text",
    "at all. A reading proposed by *that* model cannot be a memory of the edition you are checking."),
 code("def model_that_never_read(document_id: str) -> str:",
      '    """Pick the released checkpoint whose held-out digit matches this document."""',
      "    digit = str(document_id).strip()[-1]",
      f'    return f"{ORG}/Stoicheia-restoration-test{{digit}}"',
      "",
      'for phi in ["PHI 12345", "PHI 293", "TM 8100"]:',
      '    print(f"{phi:12s} -> {model_that_never_read(phi)}")',
      "",
      "# use it exactly like the backbone above",
      'REPO = model_that_never_read("PHI 293")',
      'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])',
      "sys.path.insert(0, local)",
      "restorer = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()",
      "",
      '# "[7]" is a lacuna of known width; the decree formula is deliberately incomplete',
      'text = "αγαθηιτυχηιεδοξεντ[7]βουληικαιτωιδημωι"',
      "best, width, _ = proc.restore_elastic(restorer, text, mask_dia_boundary=True)",
      'print("\\nrestored:", best)'),

 md("## 3. Tagging and parsing",
    "",
    "Four heads on one shared backbone — factored XPOS, an edit-script lemmatizer, a UPOS",
    "auxiliary and a biaffine dependency parser — all from a single forward pass."),
 code("from huggingface_hub import snapshot_download",
      f'REPO = "{ORG}/Stoicheia-tagger-parser"',
      'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])',
      "sys.path.insert(0, local)",
      "from processing_char_bert_joint import CharBertJointProcessor",
      "",
      "parser_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()",
      "jproc = CharBertJointProcessor.from_pretrained(local)",
      "",
      'words = "μῆνιν ἄειδε θεὰ Πηληϊάδεω Ἀχιλῆος".split()',
      "batch = jproc([words])",
      "with torch.no_grad():",
      "    out = parser_model(**batch)",
      "rows = jproc.decode(out, batch, ud=True)",
      "",
      "sent = rows[0] if rows and not isinstance(rows[0], dict) else rows",
      "hdr = ('id', 'form', 'lemma', 'upos', 'head', 'deprel')",
      "print('%3s %-12s%-12s%-8s%4s %s' % hdr)",
      "for i, w in enumerate(sent, 1):",
      "    print('%3d %-12s%-12s%-8s%4s %s' % (i, w['form'], w['lemma'], w['upos'], w['head'], w['deprel']))"),

 md("## 4. Vowel length and metre",
    "",
    "Greek orthography never marks vowel length: α, ι and υ — the *dichrona* — are ambiguous.",
    "Recovering it (*macronization*) is lexical knowledge, and it is the prerequisite for scanning",
    "verse. `Stoicheia-meter` does both at once; `Stoicheia-macronizer` does vowel length alone,",
    "slightly better."),
 code(f'REPO = "{ORG}/Stoicheia-meter"',
      'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])',
      "sys.path.insert(0, local)",
      "from processing_char_bert_meter import CharBertMeterProcessor",
      "",
      "meter_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()",
      "mproc = CharBertMeterProcessor()",
      "",
      'line = "ἄνδρα μοι ἔννεπε, μοῦσα, πολύτροπον, ὃς μάλα πολλὰ"',
      "batch = mproc(line)",
      "with torch.no_grad():",
      '    out = meter_model(**{k: v for k, v in batch.items() if not k.startswith("_")})',
      'print("macronized:", mproc.decode_macronization(out, batch))   # _ long, ^ short',
      'print("scanned   :", mproc.decode_scansion(out, batch))        # [heavy] {light}'),

 md("## 5. Scoring against the benchmark",
    "",
    "*Norma Syllabarum Graecarum* is a hand-annotated benchmark of macronization and",
    "syllabification. Here we score the dedicated macronizer on its test split — every ambiguous",
    "α/ι/υ position, compared against the gold mark."),
 code("import json, re",
      "from huggingface_hub import hf_hub_download",
      "",
      f'REPO = "{ORG}/Stoicheia-macronizer"',
      'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])',
      "sys.path.insert(0, local)",
      "mac_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()",
      "",
      f'path = hf_hub_download("{ORG}/norma", "data/test.jsonl", repo_type="dataset")',
      'rows = [json.loads(l) for l in open(path, encoding="utf-8")]',
      'rows = [r for r in rows if r["task"] == "macronize"][:120]   # raise for the full set',
      "",
      'MARKS = re.compile(r"[_^]")',
      "n = correct = 0",
      "for r in rows:",
      '    gold = r["text"]',
      '    raw = MARKS.sub("", gold)',
      "    batch = mproc(raw)",
      "    with torch.no_grad():",
      '        out = mac_model(**{k: v for k, v in batch.items() if not k.startswith("_")})',
      "    pred = mproc.decode_macronization(out, batch)",
      "    for g, p in zip(gold, pred):",
      "        pass",
      "    # compare mark-by-mark at the positions the gold marks",
      "    gi = pi = 0",
      "    while gi < len(gold) and pi < len(pred):",
      '        if gold[gi] in "_^" and pred[pi] in "_^":',
      "            n += 1; correct += (gold[gi] == pred[pi]); gi += 1; pi += 1",
      '        elif gold[gi] in "_^":',
      "            n += 1; gi += 1",
      '        elif pred[pi] in "_^":',
      "            pi += 1",
      "        else:",
      "            gi += 1; pi += 1",
      'print(f"macronization accuracy on {len(rows)} lines: {100*correct/max(n,1):.2f}%  ({n} scored positions)")'),

 md("---",
    "",
    "**Everything in the release**",
    "",
    "| | |",
    "|---|---|",
    "| 11 pretrained backbones | ten rotated literary folds + one documentary-clean |",
    "| 10 restoration checkpoints | one per held-out PHI/TM digit |",
    "| tagger-parser, meter, macronizer | fine-tuned from the documentary-clean backbone |",
    "| 5 datasets | pretraining corpus, synthetic augmentation, inscriptions, meter silver, benchmark |",
    "",
    "Training and evaluation code, including the split pipeline that produces the decontamination",
    "guarantee, is in the accompanying code repository."),
]

nb = {"cells": cells,
      "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
                   "language_info": {"name": "python"},
                   "colab": {"provenance": [], "toc_visible": True},
                   "accelerator": "GPU"},
      "nbformat": 4, "nbformat_minor": 0}

out = pathlib.Path("/tmp/stoicheia_hf/Stoicheia_demo.ipynb")
out.write_text(json.dumps(nb, ensure_ascii=False, indent=1), encoding="utf-8")
print("wrote", out, out.stat().st_size // 1024, "KB,", len(cells), "cells")