#!/usr/bin/env python3 """Build the Stoicheia demo notebook (anonymous, Colab-ready).""" import json, pathlib ORG = "anonymous-stoicheia" def md(*lines): return {"cell_type": "markdown", "metadata": {}, "source": [l + "\n" for l in lines]} def code(*lines): return {"cell_type": "code", "metadata": {}, "execution_count": None, "outputs": [], "source": [l + "\n" for l in lines]} cells = [ md("# Stoicheia — a character-level model for Ancient Greek", "", "Stoicheia is a 405M-parameter character-level masked-diffusion encoder for Ancient Greek.", "Its input is factored into five aligned planes — letters, word/sentence boundaries,", "diacritics, capitalization, punctuation — and **any of them can be set to *unknown* at", "inference**. One model therefore reads an edited text, bare *scriptio continua*, and a", "lacuna of unknown length, changing nothing but its input.", "", "This notebook runs the whole release end to end on a free Colab GPU (CPU works too, slower):", "", "1. restore a lacuna whose width is *not known* in advance", "2. pick the restoration model that has provably **never read** your document", "3. tag and parse a verse of Homer", "4. macronize and scan a line of verse", "5. score the macronizer against a hand-annotated benchmark", "", "Every model and dataset used below is public. Anonymous release accompanying a paper under review."), code("%pip install -q --upgrade transformers huggingface_hub safetensors torch datasets"), md("## 1. Restoring a lacuna of unknown width", "", "The hard case in epigraphy and papyrology is a break whose extent is uncertain, in text that", "carries no accents and no word division. Write `[N±M]` and the model scores every width in", "`N-M … N+M` by its own confidence, restoring the letters, the accents and the word boundaries", "together."), code("import sys, torch", "from transformers import AutoModel", "from huggingface_hub import snapshot_download", "", f'REPO = "{ORG}/Stoicheia-doc_clean" # zero exposure to inscriptions or papyri', 'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])', "sys.path.insert(0, local)", "", "model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", "from processing_char_bert import CharBertProcessor", "proc = CharBertProcessor()", "", "# John 1:1 as it would reach us on a damaged, unaccented, unspaced witness", 'damaged = "εναρχηηνο[5±3]καιολογοςηνπροστονθεον"', "best, width, candidates = proc.restore_elastic(model, damaged, mask_dia_boundary=True)", 'print("restored :", best)', 'print("width :", width, "characters")', 'for c in candidates[:5]:', ' print(" ", c)'), md("## 2. The model that has never read your document", "", "A single fixed train/test split makes a model useless for exactly the documents an editor", "cares about. Ten restoration checkpoints are released instead, one per held-out final digit", "of the PHI/TM identifier: whatever inscription or papyrus you are working on, one of the ten", "has provably never seen it during fine-tuning, and its backbone never saw a documentary text", "at all. A reading proposed by *that* model cannot be a memory of the edition you are checking."), code("def model_that_never_read(document_id: str) -> str:", ' """Pick the released checkpoint whose held-out digit matches this document."""', " digit = str(document_id).strip()[-1]", f' return f"{ORG}/Stoicheia-restoration-test{{digit}}"', "", 'for phi in ["PHI 12345", "PHI 293", "TM 8100"]:', ' print(f"{phi:12s} -> {model_that_never_read(phi)}")', "", "# use it exactly like the backbone above", 'REPO = model_that_never_read("PHI 293")', 'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])', "sys.path.insert(0, local)", "restorer = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", "", '# "[7]" is a lacuna of known width; the decree formula is deliberately incomplete', 'text = "αγαθηιτυχηιεδοξεντ[7]βουληικαιτωιδημωι"', "best, width, _ = proc.restore_elastic(restorer, text, mask_dia_boundary=True)", 'print("\\nrestored:", best)'), md("## 3. Tagging and parsing", "", "Four heads on one shared backbone — factored XPOS, an edit-script lemmatizer, a UPOS", "auxiliary and a biaffine dependency parser — all from a single forward pass."), code("from huggingface_hub import snapshot_download", f'REPO = "{ORG}/Stoicheia-tagger-parser"', 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', "sys.path.insert(0, local)", "from processing_char_bert_joint import CharBertJointProcessor", "", "parser_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", "jproc = CharBertJointProcessor.from_pretrained(local)", "", 'words = "μῆνιν ἄειδε θεὰ Πηληϊάδεω Ἀχιλῆος".split()', "batch = jproc([words])", "with torch.no_grad():", " out = parser_model(**batch)", "rows = jproc.decode(out, batch, ud=True)", "", "sent = rows[0] if rows and not isinstance(rows[0], dict) else rows", "hdr = ('id', 'form', 'lemma', 'upos', 'head', 'deprel')", "print('%3s %-12s%-12s%-8s%4s %s' % hdr)", "for i, w in enumerate(sent, 1):", " print('%3d %-12s%-12s%-8s%4s %s' % (i, w['form'], w['lemma'], w['upos'], w['head'], w['deprel']))"), md("## 4. Vowel length and metre", "", "Greek orthography never marks vowel length: α, ι and υ — the *dichrona* — are ambiguous.", "Recovering it (*macronization*) is lexical knowledge, and it is the prerequisite for scanning", "verse. `Stoicheia-meter` does both at once; `Stoicheia-macronizer` does vowel length alone,", "slightly better."), code(f'REPO = "{ORG}/Stoicheia-meter"', 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', "sys.path.insert(0, local)", "from processing_char_bert_meter import CharBertMeterProcessor", "", "meter_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", "mproc = CharBertMeterProcessor()", "", 'line = "ἄνδρα μοι ἔννεπε, μοῦσα, πολύτροπον, ὃς μάλα πολλὰ"', "batch = mproc(line)", "with torch.no_grad():", ' out = meter_model(**{k: v for k, v in batch.items() if not k.startswith("_")})', 'print("macronized:", mproc.decode_macronization(out, batch)) # _ long, ^ short', 'print("scanned :", mproc.decode_scansion(out, batch)) # [heavy] {light}'), md("## 5. Scoring against the benchmark", "", "*Norma Syllabarum Graecarum* is a hand-annotated benchmark of macronization and", "syllabification. Here we score the dedicated macronizer on its test split — every ambiguous", "α/ι/υ position, compared against the gold mark."), code("import json, re", "from huggingface_hub import hf_hub_download", "", f'REPO = "{ORG}/Stoicheia-macronizer"', 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', "sys.path.insert(0, local)", "mac_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", "", f'path = hf_hub_download("{ORG}/norma", "data/test.jsonl", repo_type="dataset")', 'rows = [json.loads(l) for l in open(path, encoding="utf-8")]', 'rows = [r for r in rows if r["task"] == "macronize"][:120] # raise for the full set', "", 'MARKS = re.compile(r"[_^]")', "n = correct = 0", "for r in rows:", ' gold = r["text"]', ' raw = MARKS.sub("", gold)', " batch = mproc(raw)", " with torch.no_grad():", ' out = mac_model(**{k: v for k, v in batch.items() if not k.startswith("_")})', " pred = mproc.decode_macronization(out, batch)", " for g, p in zip(gold, pred):", " pass", " # compare mark-by-mark at the positions the gold marks", " gi = pi = 0", " while gi < len(gold) and pi < len(pred):", ' if gold[gi] in "_^" and pred[pi] in "_^":', " n += 1; correct += (gold[gi] == pred[pi]); gi += 1; pi += 1", ' elif gold[gi] in "_^":', " n += 1; gi += 1", ' elif pred[pi] in "_^":', " pi += 1", " else:", " gi += 1; pi += 1", 'print(f"macronization accuracy on {len(rows)} lines: {100*correct/max(n,1):.2f}% ({n} scored positions)")'), md("---", "", "**Everything in the release**", "", "| | |", "|---|---|", "| 11 pretrained backbones | ten rotated literary folds + one documentary-clean |", "| 10 restoration checkpoints | one per held-out PHI/TM digit |", "| tagger-parser, meter, macronizer | fine-tuned from the documentary-clean backbone |", "| 5 datasets | pretraining corpus, synthetic augmentation, inscriptions, meter silver, benchmark |", "", "Training and evaluation code, including the split pipeline that produces the decontamination", "guarantee, is in the accompanying code repository."), ] nb = {"cells": cells, "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python"}, "colab": {"provenance": [], "toc_visible": True}, "accelerator": "GPU"}, "nbformat": 4, "nbformat_minor": 0} out = pathlib.Path("/tmp/stoicheia_hf/Stoicheia_demo.ipynb") out.write_text(json.dumps(nb, ensure_ascii=False, indent=1), encoding="utf-8") print("wrote", out, out.stat().st_size // 1024, "KB,", len(cells), "cells")