| |
| """Build the Stoicheia demo notebook (anonymous, Colab-ready).""" |
| import json, pathlib |
|
|
| ORG = "anonymous-stoicheia" |
|
|
| def md(*lines): return {"cell_type": "markdown", "metadata": {}, "source": [l + "\n" for l in lines]} |
| def code(*lines): return {"cell_type": "code", "metadata": {}, "execution_count": None, |
| "outputs": [], "source": [l + "\n" for l in lines]} |
|
|
| cells = [ |
| md("# Stoicheia — a character-level model for Ancient Greek", |
| "", |
| "Stoicheia is a 405M-parameter character-level masked-diffusion encoder for Ancient Greek.", |
| "Its input is factored into five aligned planes — letters, word/sentence boundaries,", |
| "diacritics, capitalization, punctuation — and **any of them can be set to *unknown* at", |
| "inference**. One model therefore reads an edited text, bare *scriptio continua*, and a", |
| "lacuna of unknown length, changing nothing but its input.", |
| "", |
| "This notebook runs the whole release end to end on a free Colab GPU (CPU works too, slower):", |
| "", |
| "1. restore a lacuna whose width is *not known* in advance", |
| "2. pick the restoration model that has provably **never read** your document", |
| "3. tag and parse a verse of Homer", |
| "4. macronize and scan a line of verse", |
| "5. score the macronizer against a hand-annotated benchmark", |
| "", |
| "Every model and dataset used below is public. Anonymous release accompanying a paper under review."), |
|
|
| code("%pip install -q --upgrade transformers huggingface_hub safetensors torch datasets"), |
|
|
| md("## 1. Restoring a lacuna of unknown width", |
| "", |
| "The hard case in epigraphy and papyrology is a break whose extent is uncertain, in text that", |
| "carries no accents and no word division. Write `[N±M]` and the model scores every width in", |
| "`N-M … N+M` by its own confidence, restoring the letters, the accents and the word boundaries", |
| "together."), |
| code("import sys, torch", |
| "from transformers import AutoModel", |
| "from huggingface_hub import snapshot_download", |
| "", |
| f'REPO = "{ORG}/Stoicheia-doc_clean" # zero exposure to inscriptions or papyri', |
| 'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])', |
| "sys.path.insert(0, local)", |
| "", |
| "model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", |
| "from processing_char_bert import CharBertProcessor", |
| "proc = CharBertProcessor()", |
| "", |
| "# John 1:1 as it would reach us on a damaged, unaccented, unspaced witness", |
| 'damaged = "εναρχηηνο[5±3]καιολογοςηνπροστονθεον"', |
| "best, width, candidates = proc.restore_elastic(model, damaged, mask_dia_boundary=True)", |
| 'print("restored :", best)', |
| 'print("width :", width, "characters")', |
| 'for c in candidates[:5]:', |
| ' print(" ", c)'), |
|
|
| md("## 2. The model that has never read your document", |
| "", |
| "A single fixed train/test split makes a model useless for exactly the documents an editor", |
| "cares about. Ten restoration checkpoints are released instead, one per held-out final digit", |
| "of the PHI/TM identifier: whatever inscription or papyrus you are working on, one of the ten", |
| "has provably never seen it during fine-tuning, and its backbone never saw a documentary text", |
| "at all. A reading proposed by *that* model cannot be a memory of the edition you are checking."), |
| code("def model_that_never_read(document_id: str) -> str:", |
| ' """Pick the released checkpoint whose held-out digit matches this document."""', |
| " digit = str(document_id).strip()[-1]", |
| f' return f"{ORG}/Stoicheia-restoration-test{{digit}}"', |
| "", |
| 'for phi in ["PHI 12345", "PHI 293", "TM 8100"]:', |
| ' print(f"{phi:12s} -> {model_that_never_read(phi)}")', |
| "", |
| "# use it exactly like the backbone above", |
| 'REPO = model_that_never_read("PHI 293")', |
| 'local = snapshot_download(REPO, allow_patterns=["*.py", "*.json"])', |
| "sys.path.insert(0, local)", |
| "restorer = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", |
| "", |
| '# "[7]" is a lacuna of known width; the decree formula is deliberately incomplete', |
| 'text = "αγαθηιτυχηιεδοξεντ[7]βουληικαιτωιδημωι"', |
| "best, width, _ = proc.restore_elastic(restorer, text, mask_dia_boundary=True)", |
| 'print("\\nrestored:", best)'), |
|
|
| md("## 3. Tagging and parsing", |
| "", |
| "Four heads on one shared backbone — factored XPOS, an edit-script lemmatizer, a UPOS", |
| "auxiliary and a biaffine dependency parser — all from a single forward pass."), |
| code("from huggingface_hub import snapshot_download", |
| f'REPO = "{ORG}/Stoicheia-tagger-parser"', |
| 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', |
| "sys.path.insert(0, local)", |
| "from processing_char_bert_joint import CharBertJointProcessor", |
| "", |
| "parser_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", |
| "jproc = CharBertJointProcessor.from_pretrained(local)", |
| "", |
| 'words = "μῆνιν ἄειδε θεὰ Πηληϊάδεω Ἀχιλῆος".split()', |
| "batch = jproc([words])", |
| "with torch.no_grad():", |
| " out = parser_model(**batch)", |
| "rows = jproc.decode(out, batch, ud=True)", |
| "", |
| "sent = rows[0] if rows and not isinstance(rows[0], dict) else rows", |
| "hdr = ('id', 'form', 'lemma', 'upos', 'head', 'deprel')", |
| "print('%3s %-12s%-12s%-8s%4s %s' % hdr)", |
| "for i, w in enumerate(sent, 1):", |
| " print('%3d %-12s%-12s%-8s%4s %s' % (i, w['form'], w['lemma'], w['upos'], w['head'], w['deprel']))"), |
|
|
| md("## 4. Vowel length and metre", |
| "", |
| "Greek orthography never marks vowel length: α, ι and υ — the *dichrona* — are ambiguous.", |
| "Recovering it (*macronization*) is lexical knowledge, and it is the prerequisite for scanning", |
| "verse. `Stoicheia-meter` does both at once; `Stoicheia-macronizer` does vowel length alone,", |
| "slightly better."), |
| code(f'REPO = "{ORG}/Stoicheia-meter"', |
| 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', |
| "sys.path.insert(0, local)", |
| "from processing_char_bert_meter import CharBertMeterProcessor", |
| "", |
| "meter_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", |
| "mproc = CharBertMeterProcessor()", |
| "", |
| 'line = "ἄνδρα μοι ἔννεπε, μοῦσα, πολύτροπον, ὃς μάλα πολλὰ"', |
| "batch = mproc(line)", |
| "with torch.no_grad():", |
| ' out = meter_model(**{k: v for k, v in batch.items() if not k.startswith("_")})', |
| 'print("macronized:", mproc.decode_macronization(out, batch)) # _ long, ^ short', |
| 'print("scanned :", mproc.decode_scansion(out, batch)) # [heavy] {light}'), |
|
|
| md("## 5. Scoring against the benchmark", |
| "", |
| "*Norma Syllabarum Graecarum* is a hand-annotated benchmark of macronization and", |
| "syllabification. Here we score the dedicated macronizer on its test split — every ambiguous", |
| "α/ι/υ position, compared against the gold mark."), |
| code("import json, re", |
| "from huggingface_hub import hf_hub_download", |
| "", |
| f'REPO = "{ORG}/Stoicheia-macronizer"', |
| 'local = snapshot_download(REPO, allow_patterns=["*.json", "*.txt", "*.py", "*.model"])', |
| "sys.path.insert(0, local)", |
| "mac_model = AutoModel.from_pretrained(REPO, trust_remote_code=True).eval()", |
| "", |
| f'path = hf_hub_download("{ORG}/norma", "data/test.jsonl", repo_type="dataset")', |
| 'rows = [json.loads(l) for l in open(path, encoding="utf-8")]', |
| 'rows = [r for r in rows if r["task"] == "macronize"][:120] # raise for the full set', |
| "", |
| 'MARKS = re.compile(r"[_^]")', |
| "n = correct = 0", |
| "for r in rows:", |
| ' gold = r["text"]', |
| ' raw = MARKS.sub("", gold)', |
| " batch = mproc(raw)", |
| " with torch.no_grad():", |
| ' out = mac_model(**{k: v for k, v in batch.items() if not k.startswith("_")})', |
| " pred = mproc.decode_macronization(out, batch)", |
| " for g, p in zip(gold, pred):", |
| " pass", |
| " # compare mark-by-mark at the positions the gold marks", |
| " gi = pi = 0", |
| " while gi < len(gold) and pi < len(pred):", |
| ' if gold[gi] in "_^" and pred[pi] in "_^":', |
| " n += 1; correct += (gold[gi] == pred[pi]); gi += 1; pi += 1", |
| ' elif gold[gi] in "_^":', |
| " n += 1; gi += 1", |
| ' elif pred[pi] in "_^":', |
| " pi += 1", |
| " else:", |
| " gi += 1; pi += 1", |
| 'print(f"macronization accuracy on {len(rows)} lines: {100*correct/max(n,1):.2f}% ({n} scored positions)")'), |
|
|
| md("---", |
| "", |
| "**Everything in the release**", |
| "", |
| "| | |", |
| "|---|---|", |
| "| 11 pretrained backbones | ten rotated literary folds + one documentary-clean |", |
| "| 10 restoration checkpoints | one per held-out PHI/TM digit |", |
| "| tagger-parser, meter, macronizer | fine-tuned from the documentary-clean backbone |", |
| "| 5 datasets | pretraining corpus, synthetic augmentation, inscriptions, meter silver, benchmark |", |
| "", |
| "Training and evaluation code, including the split pipeline that produces the decontamination", |
| "guarantee, is in the accompanying code repository."), |
| ] |
|
|
| nb = {"cells": cells, |
| "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, |
| "language_info": {"name": "python"}, |
| "colab": {"provenance": [], "toc_visible": True}, |
| "accelerator": "GPU"}, |
| "nbformat": 4, "nbformat_minor": 0} |
|
|
| out = pathlib.Path("/tmp/stoicheia_hf/Stoicheia_demo.ipynb") |
| out.write_text(json.dumps(nb, ensure_ascii=False, indent=1), encoding="utf-8") |
| print("wrote", out, out.stat().st_size // 1024, "KB,", len(cells), "cells") |
|
|
